<commit_before>package upload

import (
	"crypto/tls"
	"fmt"
	"os"
	"path/filepath"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/cmd/pkg/project"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstored/service/rest/upload"
	"github.com/parnurzeal/gorequest"
)

// Command contains the arguments and functions for the cli upload command.
var Command = cli.Command{
	Name:    "upload",
	Aliases: []string{"up", "u"},
	Usage:   "Upload data to MaterialsCommons",
	Flags: []cli.Flag{
		cli.IntFlag{
			Name:  "parallel, n",
			Value: 3,
			Usage: "Number of simultaneous uploads to perform, defaults to 3",
		},
	},
	Action: uploadCLI,
}

const oneMeg = 1024 * 1024
const twoMeg = oneMeg * 2
const largeFileSize = oneMeg * 25
const maxSimultaneous = 5

var proj *project.MCProject

//var pbPool = &pb.Pool{}

// uploadCLI implements the cli command upload.
func uploadCLI(c *cli.Context) {
	fmt.Println("upload: ", c.Args())
	if len(c.Args()) != 1 {
		fmt.Println("You must give the directory to walk")
		os.Exit(1)
	}
	dir := c.Args()[0]
	numThreads := getNumThreads(c)

	if !file.IsDir(dir) {
		fmt.Printf("Invalid directory: %s\n", dir)
		os.Exit(1)
	}

	uploadToServer(dir, numThreads)
}

// getNumThreads ensures that the number of parallel downloads is valid.
func getNumThreads(c *cli.Context) int {
	numThreads := c.Int("parallel")

	if numThreads < 1 {
		fmt.Println("Simultaneous downloads must be positive: ", numThreads)
		os.Exit(1)
	} else if numThreads > maxSimultaneous {
		fmt.Printf("You may not set simultaneous downloads greater than %d: %d\n", maxSimultaneous, numThreads)
		os.Exit(1)
	}

	return numThreads
}

// uploadToServer
func uploadToServer(dir string, numThreads int) {
	var err error
	proj, err = project.Find(dir)
	if err != nil {
		fmt.Println("Unable to locate project dir is in.")
		os.Exit(1)
	}
	fmt.Printf("project = '%s'\n", proj.ID)
	// _, errc := files.PWalk(dir, numThreads, processFiles)
	// if err := <-errc; err != nil {
	// 	fmt.Println("Got error: ", err)
	// }
}

// findDotMCProject will walk up from directory looking for the .mcproject
// directory. If it cannot find it, then the directory isn't in a
// known project. findProject will call os.Exit on any errors or if
// it cannot find a .mcproject directory.
func findDotMCProject(dir string) string {
	// Normalize the directory path, and convert all path separators to a
	// forward slash (/).
	dirPath, err := filepath.Abs(dir)
	if err != nil {
		fmt.Printf("Bad directory %s: %s", dir, err)
		os.Exit(1)
	}

	dirPath = filepath.ToSlash(dirPath)
	for {
		if dirPath == "/" {
			// Projects at root level not allowed
			fmt.Println("Your directory is not in a project.")
			fmt.Println("Upload a directory in a project or create a project by running the create-project command.")
			os.Exit(1)
		}

		mcprojectDir := filepath.Join(dirPath, ".mcproject")
		if file.IsDir(mcprojectDir) {
			// found it
			return mcprojectDir
		}
		dirPath = filepath.Dir(dirPath)
	}
}

// processFiles is the callback passed into PWalk. It processes each file, determines
// if it should be uploaded, and if so uploads the file. There can be a maxSimultaneous
// processFiles routines running.
func processFiles(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	fmt.Println("processFiles")
	u := &uploader{
		client: gorequest.New().TLSClientConfig(&tls.Config{InsecureSkipVerify: true}),
	}
	for entry := range entries {
		select {
		case result <- u.sendFile(entry):
		case <-done:
			// Received done, so stop processing requests.
			return
		}
	}
}

type uploader struct {
	client *gorequest.SuperAgent
}

// sendFile needs to:
//   if file is not on server then
//       send file up and send hash up at end
//           -- here we are computing hash as we send blocks up
//   else
//       compute hash
//       if hash is on server then
//          tell server to create a new entry pointing to
//          the already uploaded file
//       else
//          upload the file
//       end
//  end
//
func (u *uploader) sendFile(fileEntry files.TreeEntry) string {
	u.createUploadRequest()
	// buf := make([]byte, twoMeg)
	// f, err := os.Open(fileEntry.Path)
	// if err != nil {
	// 	return ""
	// }
	// fileHash := ""
	// // For small files just transfer the bytes. For
	// // large files compute the file hash.
	// if fileEntry.Finfo.Size() > largeFileSize {
	// 	fileHash, _ = file.HashStr(md5.New(), fileEntry.Path)
	// }
	// for {
	// 	read, err := f.Read(buf)
	// 	sendFlowChunk(buf)
	// }
	return fileEntry.Path
}

func (u *uploader) createUploadRequest() {
	fmt.Println("createUploadRequest")
	req := upload.CreateRequest{
		ProjectID:   "9ead5bbf-f7eb-4010-bc1f-e4a063f56226",
		DirectoryID: "c54a77d6-cd6d-4cd1-8f19-44facc761da6",
		FileName:    "abc.txt",
		FileSize:    10,
		FileMTime:   "Thu, 30 Apr 2015 13:10:04 EST",
	}

	var resp upload.CreateResponse
	fmt.Println("url =", app.MCApi.APIUrl("/upload"))
	r, body, errs := u.client.Post(app.MCApi.APIUrl("/upload")).Send(req).End()
	if err := app.MCApi.APIError(r, errs); err != nil {
		fmt.Println("got err from Post:", err)
		return
	}
	app.MCApi.ToJSON(body, &resp)
	fmt.Printf("%#v\n", resp)
}

func sendFlowChunk(buf []byte) {

}
<commit_msg>Add method to determine if a file may have changed (still need to compute the hash to make sure).<commit_after>package upload

import (
	"crypto/tls"
	"fmt"
	"os"
	"path/filepath"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/cmd/pkg/project"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstored/service/rest/upload"
	"github.com/parnurzeal/gorequest"
)

// Command contains the arguments and functions for the cli upload command.
var Command = cli.Command{
	Name:    "upload",
	Aliases: []string{"up", "u"},
	Usage:   "Upload data to MaterialsCommons",
	Flags: []cli.Flag{
		cli.IntFlag{
			Name:  "parallel, n",
			Value: 3,
			Usage: "Number of simultaneous uploads to perform, defaults to 3",
		},
	},
	Action: uploadCLI,
}

const oneMeg = 1024 * 1024
const twoMeg = oneMeg * 2
const largeFileSize = oneMeg * 25
const maxSimultaneous = 5

var proj *project.MCProject

//var pbPool = &pb.Pool{}

// uploadCLI implements the cli command upload.
func uploadCLI(c *cli.Context) {
	fmt.Println("upload: ", c.Args())
	if len(c.Args()) != 1 {
		fmt.Println("You must give the directory to walk")
		os.Exit(1)
	}
	dir := c.Args()[0]
	numThreads := getNumThreads(c)

	if !file.IsDir(dir) {
		fmt.Printf("Invalid directory: %s\n", dir)
		os.Exit(1)
	}

	uploadToServer(dir, numThreads)
}

// getNumThreads ensures that the number of parallel downloads is valid.
func getNumThreads(c *cli.Context) int {
	numThreads := c.Int("parallel")

	if numThreads < 1 {
		fmt.Println("Simultaneous downloads must be positive: ", numThreads)
		os.Exit(1)
	} else if numThreads > maxSimultaneous {
		fmt.Printf("You may not set simultaneous downloads greater than %d: %d\n", maxSimultaneous, numThreads)
		os.Exit(1)
	}

	return numThreads
}

// uploadToServer
func uploadToServer(dir string, numThreads int) {
	var err error
	proj, err = project.Find(dir)
	if err != nil {
		fmt.Println("Unable to locate project dir is in.")
		os.Exit(1)
	}
	fmt.Printf("project = '%s'\n", proj.ID)
	// _, errc := files.PWalk(dir, numThreads, processFiles)
	// if err := <-errc; err != nil {
	// 	fmt.Println("Got error: ", err)
	// }
}

// findDotMCProject will walk up from directory looking for the .mcproject
// directory. If it cannot find it, then the directory isn't in a
// known project. findProject will call os.Exit on any errors or if
// it cannot find a .mcproject directory.
func findDotMCProject(dir string) string {
	// Normalize the directory path, and convert all path separators to a
	// forward slash (/).
	dirPath, err := filepath.Abs(dir)
	if err != nil {
		fmt.Printf("Bad directory %s: %s", dir, err)
		os.Exit(1)
	}

	dirPath = filepath.ToSlash(dirPath)
	for {
		if dirPath == "/" {
			// Projects at root level not allowed
			fmt.Println("Your directory is not in a project.")
			fmt.Println("Upload a directory in a project or create a project by running the create-project command.")
			os.Exit(1)
		}

		mcprojectDir := filepath.Join(dirPath, ".mcproject")
		if file.IsDir(mcprojectDir) {
			// found it
			return mcprojectDir
		}
		dirPath = filepath.Dir(dirPath)
	}
}

// processFiles is the callback passed into PWalk. It processes each file, determines
// if it should be uploaded, and if so uploads the file. There can be a maxSimultaneous
// processFiles routines running.
func processFiles(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	fmt.Println("processFiles")
	u := &uploader{
		client: gorequest.New().TLSClientConfig(&tls.Config{InsecureSkipVerify: true}),
	}
	for entry := range entries {
		select {
		case result <- u.sendFile(entry):
		case <-done:
			// Received done, so stop processing requests.
			return
		}
	}
}

type uploader struct {
	client *gorequest.SuperAgent
}

// sendFile needs to:
//   if file is not on server then
//       send file up and send hash up at end
//           -- here we are computing hash as we send blocks up
//   else
//       compute hash
//       if hash is on server then
//          tell server to create a new entry pointing to
//          the already uploaded file
//       else
//          upload the file
//       end
//  end
//
func (u *uploader) sendFile(fileEntry files.TreeEntry) string {
	u.createUploadRequest()
	// buf := make([]byte, twoMeg)
	// f, err := os.Open(fileEntry.Path)
	// if err != nil {
	// 	return ""
	// }
	// fileHash := ""
	// // For small files just transfer the bytes. For
	// // large files compute the file hash.
	// if fileEntry.Finfo.Size() > largeFileSize {
	// 	fileHash, _ = file.HashStr(md5.New(), fileEntry.Path)
	// }
	// for {
	// 	read, err := f.Read(buf)
	// 	sendFlowChunk(buf)
	// }
	return fileEntry.Path
}

func fileChanged(oinfo, ninfo file.ExFileInfo) bool {
	switch {
	case oinfo.Size() != ninfo.Size():
		return true
	case oinfo.CTime().Before(ninfo.CTime()):
		return true
	case oinfo.ModTime().Before(ninfo.ModTime()):
		return true
	default:
		return false
	}
}

func (u *uploader) createUploadRequest() {
	fmt.Println("createUploadRequest")
	req := upload.CreateRequest{
		ProjectID:   "9ead5bbf-f7eb-4010-bc1f-e4a063f56226",
		DirectoryID: "c54a77d6-cd6d-4cd1-8f19-44facc761da6",
		FileName:    "abc.txt",
		FileSize:    10,
		FileMTime:   "Thu, 30 Apr 2015 13:10:04 EST",
	}

	var resp upload.CreateResponse
	fmt.Println("url =", app.MCApi.APIUrl("/upload"))
	r, body, errs := u.client.Post(app.MCApi.APIUrl("/upload")).Send(req).End()
	if err := app.MCApi.APIError(r, errs); err != nil {
		fmt.Println("got err from Post:", err)
		return
	}
	app.MCApi.ToJSON(body, &resp)
	fmt.Printf("%#v\n", resp)
}

func sendFlowChunk(buf []byte) {

}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"flag"
	"fmt"
	"time"

	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
)

func main() {
	flag.Parse()
	if err := embd.InitSPI(); err != nil {
		panic(err)
	}
	defer embd.CloseSPI()

	bus := embd.NewSPIBus(embd.SpiMode0, 0, 1000000, 8, 0)
	defer bus.Close()

	for i := 0; i < 30; i++ {
		time.Sleep(1 * time.Second)
		val, _ := getSensorValue(bus)
		fmt.Printf("value is: %v\n", val)
	}

}

func getSensorValue(bus embd.SPIBus) (uint16, error) {
	data := make([]uint8, 3)
	data[0] = 1
	data[1] = 128
	data[2] = 0
	var err error
	err = bus.TransferAndRecieveData(data)
	if err != nil {
		return uint16(0), err
	}
	return uint16(data[1]&0x03)<<8 | uint16(data[2]), nil
}
<commit_msg>spi: removing blank line<commit_after>// +build ignore

package main

import (
	"flag"
	"fmt"
	"time"

	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
)

func main() {
	flag.Parse()
	if err := embd.InitSPI(); err != nil {
		panic(err)
	}
	defer embd.CloseSPI()

	bus := embd.NewSPIBus(embd.SpiMode0, 0, 1000000, 8, 0)
	defer bus.Close()

	for i := 0; i < 30; i++ {
		time.Sleep(1 * time.Second)
		val, _ := getSensorValue(bus)
		fmt.Printf("value is: %v\n", val)
	}
}

func getSensorValue(bus embd.SPIBus) (uint16, error) {
	data := make([]uint8, 3)
	data[0] = 1
	data[1] = 128
	data[2] = 0
	var err error
	err = bus.TransferAndRecieveData(data)
	if err != nil {
		return uint16(0), err
	}
	return uint16(data[1]&0x03)<<8 | uint16(data[2]), nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/google/gxui"
	"github.com/google/gxui/drivers/gl"
	"github.com/google/gxui/samples/flags"
)

func appMain(driver gxui.Driver) {
	theme := flags.CreateTheme(driver)

	label1 := theme.CreateLabel()
	label1.SetColor(gxui.White)
	label1.SetText("1x1")

	cell1x1 := theme.CreateLinearLayout()
	cell1x1.SetBackgroundBrush(gxui.CreateBrush(gxui.Blue40))
	cell1x1.SetHorizontalAlignment(gxui.AlignCenter)
	cell1x1.AddChild(label1)

	label2 := theme.CreateLabel()
	label2.SetColor(gxui.White)
	label2.SetText("2x1")

	cell2x1 := theme.CreateLinearLayout()
	cell2x1.SetBackgroundBrush(gxui.CreateBrush(gxui.Green40))
	cell2x1.SetHorizontalAlignment(gxui.AlignCenter)
	cell2x1.AddChild(label2)

	label3 := theme.CreateLabel()
	label3.SetColor(gxui.White)
	label3.SetText("1x2")

	cell1x2 := theme.CreateLinearLayout()
	cell1x2.SetBackgroundBrush(gxui.CreateBrush(gxui.Red40))
	cell1x2.SetHorizontalAlignment(gxui.AlignCenter)
	cell1x2.AddChild(label3)

	table := theme.CreateTableLayout()
	table.SetGrid(3, 2) // rows, columns

	// row, column, horizontal span, vertical span
	table.SetChildAt(0, 0, 1, 1, cell1x1)
	table.SetChildAt(0, 1, 2, 1, cell2x1)
	table.SetChildAt(2, 0, 1, 2, cell1x2)

	window := theme.CreateWindow(800, 600, "Table")
	window.AddChild(table)
	window.OnClose(driver.Terminate)
}

func main() {
	gl.StartDriver(appMain)
}
<commit_msg>Changed comment to show columns, rows in the correct order for SetGrid() in sample.<commit_after>package main

import (
	"github.com/google/gxui"
	"github.com/google/gxui/drivers/gl"
	"github.com/google/gxui/samples/flags"
)

func appMain(driver gxui.Driver) {
	theme := flags.CreateTheme(driver)

	label1 := theme.CreateLabel()
	label1.SetColor(gxui.White)
	label1.SetText("1x1")

	cell1x1 := theme.CreateLinearLayout()
	cell1x1.SetBackgroundBrush(gxui.CreateBrush(gxui.Blue40))
	cell1x1.SetHorizontalAlignment(gxui.AlignCenter)
	cell1x1.AddChild(label1)

	label2 := theme.CreateLabel()
	label2.SetColor(gxui.White)
	label2.SetText("2x1")

	cell2x1 := theme.CreateLinearLayout()
	cell2x1.SetBackgroundBrush(gxui.CreateBrush(gxui.Green40))
	cell2x1.SetHorizontalAlignment(gxui.AlignCenter)
	cell2x1.AddChild(label2)

	label3 := theme.CreateLabel()
	label3.SetColor(gxui.White)
	label3.SetText("1x2")

	cell1x2 := theme.CreateLinearLayout()
	cell1x2.SetBackgroundBrush(gxui.CreateBrush(gxui.Red40))
	cell1x2.SetHorizontalAlignment(gxui.AlignCenter)
	cell1x2.AddChild(label3)

	table := theme.CreateTableLayout()
	table.SetGrid(3, 2) // columns, rows

	// row, column, horizontal span, vertical span
	table.SetChildAt(0, 0, 1, 1, cell1x1)
	table.SetChildAt(0, 1, 2, 1, cell2x1)
	table.SetChildAt(2, 0, 1, 2, cell1x2)

	window := theme.CreateWindow(800, 600, "Table")
	window.AddChild(table)
	window.OnClose(driver.Terminate)
}

func main() {
	gl.StartDriver(appMain)
}
<|endoftext|>


<commit_before>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package rand

import "io"
import "bytes"
import "compress/flate"
import "testing"
import "github.com/stretchr/testify/assert"

func TestRead(t *testing.T) {
	var n int = 1 << 24
	if testing.Short() {
		n = 1 << 16
	}

	d := make([]byte, n)
	n, err := io.ReadFull(Reader, d)
	assert.Equal(t, len(d), n)
	assert.Nil(t, err)

	var b bytes.Buffer
	z, _ := flate.NewWriter(&b, 5)
	z.Write(d)
	z.Close()
	assert.True(t, b.Len() >= len(d)*99/100)
}

func TestReadEmpty(t *testing.T) {
	n, err := Reader.Read(make([]byte, 0))
	assert.Equal(t, 0, n)
	assert.Nil(t, err)

	n, err = Reader.Read(nil)
	assert.Equal(t, 0, n)
	assert.Nil(t, err)
}
<commit_msg>Trivial comment change<commit_after>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package rand

import "io"
import "bytes"
import "compress/flate"
import "testing"
import "github.com/stretchr/testify/assert"

func TestRead(t *testing.T) {
	var n int = 1 << 24 // 16 MiB
	if testing.Short() {
		n = 1 << 16 // 64 KiB
	}

	d := make([]byte, n)
	n, err := io.ReadFull(Reader, d)
	assert.Equal(t, len(d), n)
	assert.Nil(t, err)

	var b bytes.Buffer
	z, _ := flate.NewWriter(&b, 5)
	z.Write(d)
	z.Close()
	assert.True(t, b.Len() >= len(d)*99/100)
}

func TestReadEmpty(t *testing.T) {
	n, err := Reader.Read(make([]byte, 0))
	assert.Equal(t, 0, n)
	assert.Nil(t, err)

	n, err = Reader.Read(nil)
	assert.Equal(t, 0, n)
	assert.Nil(t, err)
}
<|endoftext|>


<commit_before>package templates

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"text/template"
)

const (
	defaultExtensions string = ".thtml .html .js .css"
)

// Service wraps multiple templates within a directory
type Service struct {
	sync.Mutex

	// Filename extensions supported
	exts []string

	// Templates directory
	tplDir string

	// Build input directory
	publicDir string

	// Build output
	buildDir string

	// Template wrapper
	tpl *template.Template
}

// Load creates a new *templates.Service object and loads the templates in the provided directory.
// Custom set of filename extensions can be supplied
func Load(dir string, extensions ...string) (*Service, error) {
	s := new(Service)

	if len(extensions) == 0 {
		extensions = strings.Split(defaultExtensions, " ")
	}
	for _, ext := range extensions {
		s.AddExtension(ext)
	}

	err := s.Load(dir)
	if err != nil {
		return nil, err
	}

	return s, nil
}

// AddExtension adds a new filename extension (i.e. ".txt") to the list of extensions to support.
// Extensions not supported will be rendered and/or compiled as they are without template parsing.
// This method is safe to use from multiple/concurrent goroutines.
func (s *Service) AddExtension(ext string) {
	// Sync
	s.Lock()
	defer s.Unlock()

	// Check
	if s.exts == nil {
		s.exts = make([]string, 0)
	}

	// Avoid repeated
	for i := range s.exts {
		if s.exts[i] == ext {
			return
		}
	}

	// Add
	s.exts = append(s.exts, strings.TrimSpace(ext))
}

// RemoveExtension deletes a supported filename extension from the list.
// This method is safe to use from multiple/concurrent goroutines.
func (s *Service) RemoveExtension(ext string) {
	s.Lock()
	defer s.Unlock()

	for i := range s.exts {
		if s.exts[i] == ext {
			s.exts = append(s.exts[:i], s.exts[i+1:]...)
			return
		}
	}
}

// ValidExtension returns true if the filename extension provided is supported.
// This method is safe to use from multiple/concurrent goroutines.
func (s *Service) ValidExtension(ext string) bool {
	s.Lock()
	defer s.Unlock()

	for _, e := range s.exts {
		if e == ext {
			return true
		}
	}

	return false
}

// Load takes a directory path and loads all templates on it.
// This method is NOT safe to use from multiple/concurrent goroutines
func (s *Service) Load(dir string) error {
	var err error

	// Parse dir name
	s.tplDir, err = filepath.Abs(dir)
	if err != nil {
		return err
	}

	// Init template
	s.tpl = template.New(s.tplDir)

	// Load
	err = filepath.Walk(s.tplDir, s.loadFn)
	if err != nil {
		return err
	}

	return nil
}

func (s *Service) loadFn(path string, info os.FileInfo, err error) error {
	if !info.IsDir() && s.ValidExtension(filepath.Ext(path)) {
		// Load content
		content, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}

		// Set tpl name
		n := strings.TrimPrefix(path, s.tplDir)
		for len(n) > 0 && n[0] == '/' {
			n = n[1:]
		}

		// Load template.
		_, err = s.tpl.New(n).Parse(string(content))
		if err != nil {
			return err
		}
	}

	return nil
}

// Render compiles the provided template filename in the loaded templates and writes the output to the provided io.Writer.
// This method is safe to use from multiple/concurrent goroutines
func (s *Service) Render(w io.Writer, filename string, data interface{}) error {
	// Check load
	s.Lock()
	empty := (s.tpl == nil)
	s.Unlock()
	if empty {
		return NewEmptyTemplateError()
	}

	// Load content
	fn, err := filepath.Abs(filename)
	if err != nil {
		return err
	}
	content, err := ioutil.ReadFile(fn)
	if err != nil {
		return err
	}

	// Create buffer
	buff := new(bytes.Buffer)

	if s.ValidExtension(filepath.Ext(fn)) {
		// Copy template object
		s.Lock()
		tmpTpl, err := s.tpl.Clone()
		s.Unlock()
		if err != nil {
			return err
		}

		// Parse template
		_, err = tmpTpl.New(fn).Parse(string(content))
		if err != nil {
			return err
		}

		// Execute template
		err = tmpTpl.ExecuteTemplate(buff, fn, data)
		if err != nil {
			return err
		}
	} else {
		buff.Write(content)
	}

	// Flush buffer
	_, err = w.Write(buff.Bytes())
	return err
}

// Build compiles all files in the provided directory and outputs the results to the build dir.
// This method is NOT safe to use from multiple/concurrent goroutines
func (s *Service) Build(in, out string) (err error) {
	if s.tpl == nil {
		return NewEmptyTemplateError()
	}

	s.publicDir, err = filepath.Abs(in)
	if err != nil {
		return err
	}
	s.buildDir, err = filepath.Abs(out)
	if err != nil {
		return err
	}

	err = filepath.Walk(s.publicDir, s.buildFn)
	if err != nil {
		return err
	}

	return nil
}

func (s *Service) buildFn(filename string, info os.FileInfo, err error) error {
	// Ensure directories
	if !info.IsDir() {
		// Create output
		in := strings.TrimPrefix(filename, s.publicDir)
		out, err := filepath.Abs(path.Join(s.buildDir, in))
		if err != nil {
			return err
		}

		err = os.MkdirAll(path.Dir(out), 0755)
		if err != nil {
			return err
		}

		// Open file
		f, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE, 0755)
		if err != nil {
			return err
		}
		defer f.Close()

		// Render

		return s.Render(f, filename, nil)
	}

	return nil
}
<commit_msg>More comments<commit_after>// Package templates compiles all templates in a directory tree into a Service
// that provides a Render method to execute other templates inside its context.
//
// Example
//
// The following example program will load a template directory tree defined as a constant
// and then render a template file from another directory using the loaded Service into the standard output.
//
//  package main
//
//  import (
//      "os"
//      "path"
//      "github.com/leonelquinteros/thtml/templates"
//  )
//
//  const (
//      _templates = "/path/to/templates"
//		_public    = "/path/to/web/root"
//  )
//
//  func main() {
//      tplService, err := templates.Load(_templates, ".html,.css,.js")
//      if err != nil {
//          panic(err.Error())
//      }
//
//      tplService.Render(os.Stdout, path.Join(_public, "index.html"), nil)
//  }
//
package templates

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"text/template"
)

const (
	defaultExtensions string = ".thtml .html .js .css"
)

// Service is the template handler.
// After Load()'ing a directory tree, will be ready to render any template into its context.
type Service struct {
	sync.Mutex

	// Filename extensions supported
	exts []string

	// Templates directory
	tplDir string

	// Build input directory
	publicDir string

	// Build output
	buildDir string

	// Template wrapper
	tpl *template.Template
}

// Load creates a new *templates.Service object and loads the templates in the provided directory.
// Custom set of filename extensions can be supplied
func Load(dir string, extensions ...string) (*Service, error) {
	s := new(Service)

	if len(extensions) == 0 {
		extensions = strings.Split(defaultExtensions, " ")
	}
	for _, ext := range extensions {
		s.AddExtension(ext)
	}

	err := s.Load(dir)
	if err != nil {
		return nil, err
	}

	return s, nil
}

// AddExtension adds a new filename extension (i.e. ".txt") to the list of extensions to support.
// Extensions not supported will be rendered and/or compiled as they are without template parsing.
// This method is safe to use from multiple/concurrent goroutines.
func (s *Service) AddExtension(ext string) {
	// Sync
	s.Lock()
	defer s.Unlock()

	// Check
	if s.exts == nil {
		s.exts = make([]string, 0)
	}

	// Avoid repeated
	for i := range s.exts {
		if s.exts[i] == ext {
			return
		}
	}

	// Add
	s.exts = append(s.exts, strings.TrimSpace(ext))
}

// RemoveExtension deletes a supported filename extension from the list.
// This method is safe to use from multiple/concurrent goroutines.
func (s *Service) RemoveExtension(ext string) {
	s.Lock()
	defer s.Unlock()

	for i := range s.exts {
		if s.exts[i] == ext {
			s.exts = append(s.exts[:i], s.exts[i+1:]...)
			return
		}
	}
}

// ValidExtension returns true if the filename extension provided is supported.
// This method is safe to use from multiple/concurrent goroutines.
func (s *Service) ValidExtension(ext string) bool {
	s.Lock()
	defer s.Unlock()

	for _, e := range s.exts {
		if e == ext {
			return true
		}
	}

	return false
}

// Load takes a directory path and loads all templates on it.
// This method is NOT safe to use from multiple/concurrent goroutines
func (s *Service) Load(dir string) error {
	var err error

	// Parse dir name
	s.tplDir, err = filepath.Abs(dir)
	if err != nil {
		return err
	}

	// Init template
	s.tpl = template.New(s.tplDir)

	// Load
	err = filepath.Walk(s.tplDir, s.loadFn)
	if err != nil {
		return err
	}

	return nil
}

func (s *Service) loadFn(path string, info os.FileInfo, err error) error {
	if !info.IsDir() && s.ValidExtension(filepath.Ext(path)) {
		// Load content
		content, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}

		// Set tpl name
		n := strings.TrimPrefix(path, s.tplDir)
		for len(n) > 0 && n[0] == '/' {
			n = n[1:]
		}

		// Load template.
		_, err = s.tpl.New(n).Parse(string(content))
		if err != nil {
			return err
		}
	}

	return nil
}

// Render compiles the provided template filename in the loaded templates and writes the output to the provided io.Writer.
// This method is safe to use from multiple/concurrent goroutines
func (s *Service) Render(w io.Writer, filename string, data interface{}) error {
	// Check load
	s.Lock()
	empty := (s.tpl == nil)
	s.Unlock()
	if empty {
		return NewEmptyTemplateError()
	}

	// Load content
	fn, err := filepath.Abs(filename)
	if err != nil {
		return err
	}
	content, err := ioutil.ReadFile(fn)
	if err != nil {
		return err
	}

	// Create buffer
	buff := new(bytes.Buffer)

	if s.ValidExtension(filepath.Ext(fn)) {
		// Copy template object
		s.Lock()
		tmpTpl, err := s.tpl.Clone()
		s.Unlock()
		if err != nil {
			return err
		}

		// Parse template
		_, err = tmpTpl.New(fn).Parse(string(content))
		if err != nil {
			return err
		}

		// Execute template
		err = tmpTpl.ExecuteTemplate(buff, fn, data)
		if err != nil {
			return err
		}
	} else {
		buff.Write(content)
	}

	// Flush buffer
	_, err = w.Write(buff.Bytes())
	return err
}

// Build compiles all files in the provided directory and outputs the results to the build dir.
// This method is NOT safe to use from multiple/concurrent goroutines
func (s *Service) Build(in, out string) (err error) {
	if s.tpl == nil {
		return NewEmptyTemplateError()
	}

	s.publicDir, err = filepath.Abs(in)
	if err != nil {
		return err
	}
	s.buildDir, err = filepath.Abs(out)
	if err != nil {
		return err
	}

	err = filepath.Walk(s.publicDir, s.buildFn)
	if err != nil {
		return err
	}

	return nil
}

func (s *Service) buildFn(filename string, info os.FileInfo, err error) error {
	// Ensure directories
	if !info.IsDir() {
		// Create output
		in := strings.TrimPrefix(filename, s.publicDir)
		out, err := filepath.Abs(path.Join(s.buildDir, in))
		if err != nil {
			return err
		}

		err = os.MkdirAll(path.Dir(out), 0755)
		if err != nil {
			return err
		}

		// Open file
		f, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE, 0755)
		if err != nil {
			return err
		}
		defer f.Close()

		// Render

		return s.Render(f, filename, nil)
	}

	return nil
}
<|endoftext|>


<commit_before>package irma

import (
	goerrors "errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/big"
)

type (
	// PrivateKeyRing provides access to a set of private keys.
	PrivateKeyRing interface {
		// Latest returns the private key with the highest counter for the specified issuer, if any,
		// or an error.
		Latest(id IssuerIdentifier) (*gabi.PrivateKey, error)

		// Get returns the specified private key, or an error.
		Get(id IssuerIdentifier, counter uint) (*gabi.PrivateKey, error)

		// Iterate executes the specified function on each private key of the specified issuer
		// present in the ring. The private keys are offered to the function in no particular order,
		// and the same key may be offered multiple times. Returns on the first error returned
		// by the function.
		Iterate(id IssuerIdentifier, f func(sk *gabi.PrivateKey) error) error
	}

	// PrivateKeyRingFolder represents a folder on disk containing private keys with filenames
	// of the form scheme.issuer.xml and scheme.issuer.counter.xml.
	PrivateKeyRingFolder struct {
		path string
		conf *Configuration
	}

	// privateKeyRingScheme provides access to private keys present in a scheme.
	privateKeyRingScheme struct {
		conf *Configuration
	}

	// privateKeyRingMerge is a merge of multiple key rings into one, provides access to the
	// private keys of all of them.
	privateKeyRingMerge struct {
		rings []PrivateKeyRing
	}
)

var (
	ErrMissingPrivateKey = fmt.Errorf("issuer private key not found: %w", os.ErrNotExist)
)

func NewPrivateKeyRingFolder(path string, conf *Configuration) (*PrivateKeyRingFolder, error) {
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}
	ring := &PrivateKeyRingFolder{path, conf}
	for _, file := range files {
		filename := file.Name()
		issuerid, counter, err := ring.parseFilename(filename)
		if err != nil {
			return nil, err
		}
		if issuerid == nil {
			Logger.WithField("file", filename).Infof("Skipping non-private key file encountered in private keys path")
			continue
		}
		sk, err := ring.readFile(filename, *issuerid)
		if err != nil {
			return nil, err
		}
		if counter != nil && *counter != sk.Counter {
			return nil, errors.Errorf("private key %s has wrong counter %d in filename, should be %d", filename, counter, sk.Counter)
		}
	}
	return ring, nil
}

func (_ *PrivateKeyRingFolder) parseFilename(filename string) (*IssuerIdentifier, *uint, error) {
	// This regexp returns one of the following:
	// [ "foo.bar.xml", "foo.bar", "", "" ] in case of "foo.bar.xml"
	// [ "foo.bar.xml", "foo.bar", ".2", "2" ] in case of "foo.bar.2.xml"
	// nil in case of other files.
	matches := regexp.MustCompile(`^([^.]+\.[^.]+)(\.(\d+))?\.xml$`).FindStringSubmatch(filename)

	if len(matches) != 4 {
		return nil, nil, nil
	}
	issuerid := NewIssuerIdentifier(matches[1])
	if matches[3] == "" {
		return &issuerid, nil, nil
	}
	counter, err := strconv.ParseUint(matches[3], 10, 32)
	if err != nil {
		return nil, nil, err
	}
	c := uint(counter)
	return &issuerid, &c, nil
}

func (p *PrivateKeyRingFolder) readFile(filename string, id IssuerIdentifier) (*gabi.PrivateKey, error) {
	scheme := p.conf.SchemeManagers[id.SchemeManagerIdentifier()]
	if scheme == nil {
		return nil, errors.Errorf("Private key of issuer %s belongs to unknown scheme", id.String())
	}
	sk, err := gabi.NewPrivateKeyFromFile(filepath.Join(p.path, filename), scheme.Demo)
	if err != nil {
		return nil, err
	}
	if err = validatePrivateKey(id, sk, p.conf); err != nil {
		return nil, err
	}
	return sk, nil
}

func (p *PrivateKeyRingFolder) Get(id IssuerIdentifier, counter uint) (*gabi.PrivateKey, error) {
	sk, err := p.readFile(fmt.Sprintf("%s.%d.xml", id.String(), counter), id)
	if err != nil && !goerrors.Is(err, os.ErrNotExist) {
		return nil, err
	}
	if sk != nil {
		return sk, nil
	}
	sk, err = p.readFile(fmt.Sprintf("%s.xml", id.String()), id)
	if err != nil {
		return nil, err
	}
	if counter != sk.Counter {
		return nil, ErrMissingPrivateKey
	}
	return sk, nil
}

func (p *PrivateKeyRingFolder) Latest(id IssuerIdentifier) (*gabi.PrivateKey, error) {
	var sk *gabi.PrivateKey
	if err := p.Iterate(id, func(s *gabi.PrivateKey) error {
		if sk == nil || s.Counter > sk.Counter {
			sk = s
		}
		return nil
	}); err != nil {
		return nil, err
	}
	if sk == nil {
		return nil, ErrMissingPrivateKey
	}
	return sk, nil
}

func (p *PrivateKeyRingFolder) Iterate(id IssuerIdentifier, f func(sk *gabi.PrivateKey) error) error {
	files, err := filepath.Glob(filepath.Join(p.path, fmt.Sprintf("%s*", id.String())))
	if err != nil {
		return err
	}
	for _, file := range files {
		sk, err := p.readFile(filepath.Base(file), id)
		if err != nil {
			return err
		}
		if err = f(sk); err != nil {
			return err
		}
	}
	return nil
}

func newPrivateKeyRingScheme(conf *Configuration) (*privateKeyRingScheme, error) {
	ring := &privateKeyRingScheme{conf}
	if err := validatePrivateKeyRing(ring, conf); err != nil {
		return nil, err
	}
	return ring, nil
}

func (p *privateKeyRingScheme) counters(issuerid IssuerIdentifier) (i []uint, err error) {
	scheme := p.conf.SchemeManagers[issuerid.SchemeManagerIdentifier()]
	return matchKeyPattern(filepath.Join(scheme.path(), issuerid.Name(), "PrivateKeys", "*"))
}

func (p *privateKeyRingScheme) Get(id IssuerIdentifier, counter uint) (*gabi.PrivateKey, error) {
	schemeID := id.SchemeManagerIdentifier()
	scheme := p.conf.SchemeManagers[schemeID]
	if scheme == nil {
		return nil, errors.Errorf("Private key of issuer %s belongs to unknown scheme", id.String())
	}
	file := filepath.Join(scheme.path(), id.Name(), "PrivateKeys", strconv.FormatUint(uint64(counter), 10)+".xml")
	sk, err := gabi.NewPrivateKeyFromFile(file, scheme.Demo)
	if err != nil {
		return nil, err
	}
	if sk.Counter != counter {
		return nil, errors.Errorf("Private key %s of issuer %s has wrong <Counter>", file, id.String())
	}
	if err = validatePrivateKey(id, sk, p.conf); err != nil {
		return nil, err
	}
	return sk, nil
}

func (p *privateKeyRingScheme) Latest(id IssuerIdentifier) (*gabi.PrivateKey, error) {
	counters, err := p.counters(id)
	if err != nil {
		return nil, err
	}
	if len(counters) == 0 {
		return nil, ErrMissingPrivateKey
	}
	return p.Get(id, counters[len(counters)-1])
}

func (p *privateKeyRingScheme) Iterate(id IssuerIdentifier, f func(sk *gabi.PrivateKey) error) error {
	indices, err := p.counters(id)
	if err != nil {
		return err
	}
	for _, counter := range indices {
		sk, err := p.Get(id, counter)
		if err != nil {
			return err
		}
		if err = f(sk); err != nil {
			return err
		}
	}
	return nil
}

func (p *privateKeyRingMerge) Add(ring PrivateKeyRing) {
	p.rings = append(p.rings, ring)
}

func (p *privateKeyRingMerge) Get(id IssuerIdentifier, counter uint) (*gabi.PrivateKey, error) {
	for _, ring := range p.rings {
		sk, err := ring.Get(id, counter)
		if err == nil {
			return sk, nil
		}
		if !goerrors.Is(err, os.ErrNotExist) {
			return nil, err
		}
	}
	return nil, ErrMissingPrivateKey
}

func (p *privateKeyRingMerge) Latest(id IssuerIdentifier) (*gabi.PrivateKey, error) {
	var sk *gabi.PrivateKey
	for _, ring := range p.rings {
		s, err := ring.Latest(id)
		if err != nil && !goerrors.Is(err, os.ErrNotExist) {
			return nil, err
		}
		if s != nil && (sk == nil || s.Counter > sk.Counter) {
			sk = s
		}
	}
	if sk == nil {
		return nil, ErrMissingPrivateKey
	}
	return sk, nil
}

func (p *privateKeyRingMerge) Iterate(id IssuerIdentifier, f func(sk *gabi.PrivateKey) error) error {
	for _, ring := range p.rings {
		if err := ring.Iterate(id, f); err != nil {
			return err
		}
	}
	return nil
}

func validatePrivateKey(issuerid IssuerIdentifier, sk *gabi.PrivateKey, conf *Configuration) error {
	if _, ok := conf.Issuers[issuerid]; !ok {
		return errors.Errorf("Private key %d of issuer %s belongs to an unknown issuer", sk.Counter, issuerid.String())
	}
	pk, err := conf.PublicKey(issuerid, sk.Counter)
	if err != nil {
		return err
	}
	if pk == nil {
		return errors.Errorf("Private key %d of issuer %s has no corresponding public key", sk.Counter, issuerid.String())
	}
	if new(big.Int).Mul(sk.P, sk.Q).Cmp(pk.N) != 0 {
		return errors.Errorf("Private key %d of issuer %s does not belong to corresponding public key", sk.Counter, issuerid.String())
	}
	if sk.RevocationSupported() != pk.RevocationSupported() {
		return errors.Errorf("revocation support of private key %d of issuer %s is not consistent with corresponding public key", sk.Counter, issuerid.String())
	}
	return nil
}

func validatePrivateKeyRing(ring PrivateKeyRing, conf *Configuration) error {
	for issuerid := range conf.Issuers {
		err := ring.Iterate(issuerid, func(sk *gabi.PrivateKey) error {
			return validatePrivateKey(issuerid, sk, conf)
		})
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>fix: make inconsistent revocation support in private/public keys not fatal in case of demo schemes<commit_after>package irma

import (
	goerrors "errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/big"
)

type (
	// PrivateKeyRing provides access to a set of private keys.
	PrivateKeyRing interface {
		// Latest returns the private key with the highest counter for the specified issuer, if any,
		// or an error.
		Latest(id IssuerIdentifier) (*gabi.PrivateKey, error)

		// Get returns the specified private key, or an error.
		Get(id IssuerIdentifier, counter uint) (*gabi.PrivateKey, error)

		// Iterate executes the specified function on each private key of the specified issuer
		// present in the ring. The private keys are offered to the function in no particular order,
		// and the same key may be offered multiple times. Returns on the first error returned
		// by the function.
		Iterate(id IssuerIdentifier, f func(sk *gabi.PrivateKey) error) error
	}

	// PrivateKeyRingFolder represents a folder on disk containing private keys with filenames
	// of the form scheme.issuer.xml and scheme.issuer.counter.xml.
	PrivateKeyRingFolder struct {
		path string
		conf *Configuration
	}

	// privateKeyRingScheme provides access to private keys present in a scheme.
	privateKeyRingScheme struct {
		conf *Configuration
	}

	// privateKeyRingMerge is a merge of multiple key rings into one, provides access to the
	// private keys of all of them.
	privateKeyRingMerge struct {
		rings []PrivateKeyRing
	}
)

var (
	ErrMissingPrivateKey = fmt.Errorf("issuer private key not found: %w", os.ErrNotExist)
)

func NewPrivateKeyRingFolder(path string, conf *Configuration) (*PrivateKeyRingFolder, error) {
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}
	ring := &PrivateKeyRingFolder{path, conf}
	for _, file := range files {
		filename := file.Name()
		issuerid, counter, err := ring.parseFilename(filename)
		if err != nil {
			return nil, err
		}
		if issuerid == nil {
			Logger.WithField("file", filename).Infof("Skipping non-private key file encountered in private keys path")
			continue
		}
		sk, err := ring.readFile(filename, *issuerid)
		if err != nil {
			return nil, err
		}
		if counter != nil && *counter != sk.Counter {
			return nil, errors.Errorf("private key %s has wrong counter %d in filename, should be %d", filename, counter, sk.Counter)
		}
	}
	return ring, nil
}

func (_ *PrivateKeyRingFolder) parseFilename(filename string) (*IssuerIdentifier, *uint, error) {
	// This regexp returns one of the following:
	// [ "foo.bar.xml", "foo.bar", "", "" ] in case of "foo.bar.xml"
	// [ "foo.bar.xml", "foo.bar", ".2", "2" ] in case of "foo.bar.2.xml"
	// nil in case of other files.
	matches := regexp.MustCompile(`^([^.]+\.[^.]+)(\.(\d+))?\.xml$`).FindStringSubmatch(filename)

	if len(matches) != 4 {
		return nil, nil, nil
	}
	issuerid := NewIssuerIdentifier(matches[1])
	if matches[3] == "" {
		return &issuerid, nil, nil
	}
	counter, err := strconv.ParseUint(matches[3], 10, 32)
	if err != nil {
		return nil, nil, err
	}
	c := uint(counter)
	return &issuerid, &c, nil
}

func (p *PrivateKeyRingFolder) readFile(filename string, id IssuerIdentifier) (*gabi.PrivateKey, error) {
	scheme := p.conf.SchemeManagers[id.SchemeManagerIdentifier()]
	if scheme == nil {
		return nil, errors.Errorf("Private key of issuer %s belongs to unknown scheme", id.String())
	}
	sk, err := gabi.NewPrivateKeyFromFile(filepath.Join(p.path, filename), scheme.Demo)
	if err != nil {
		return nil, err
	}
	if err = validatePrivateKey(id, sk, p.conf); err != nil {
		return nil, err
	}
	return sk, nil
}

func (p *PrivateKeyRingFolder) Get(id IssuerIdentifier, counter uint) (*gabi.PrivateKey, error) {
	sk, err := p.readFile(fmt.Sprintf("%s.%d.xml", id.String(), counter), id)
	if err != nil && !goerrors.Is(err, os.ErrNotExist) {
		return nil, err
	}
	if sk != nil {
		return sk, nil
	}
	sk, err = p.readFile(fmt.Sprintf("%s.xml", id.String()), id)
	if err != nil {
		return nil, err
	}
	if counter != sk.Counter {
		return nil, ErrMissingPrivateKey
	}
	return sk, nil
}

func (p *PrivateKeyRingFolder) Latest(id IssuerIdentifier) (*gabi.PrivateKey, error) {
	var sk *gabi.PrivateKey
	if err := p.Iterate(id, func(s *gabi.PrivateKey) error {
		if sk == nil || s.Counter > sk.Counter {
			sk = s
		}
		return nil
	}); err != nil {
		return nil, err
	}
	if sk == nil {
		return nil, ErrMissingPrivateKey
	}
	return sk, nil
}

func (p *PrivateKeyRingFolder) Iterate(id IssuerIdentifier, f func(sk *gabi.PrivateKey) error) error {
	files, err := filepath.Glob(filepath.Join(p.path, fmt.Sprintf("%s*", id.String())))
	if err != nil {
		return err
	}
	for _, file := range files {
		sk, err := p.readFile(filepath.Base(file), id)
		if err != nil {
			return err
		}
		if err = f(sk); err != nil {
			return err
		}
	}
	return nil
}

func newPrivateKeyRingScheme(conf *Configuration) (*privateKeyRingScheme, error) {
	ring := &privateKeyRingScheme{conf}
	if err := validatePrivateKeyRing(ring, conf); err != nil {
		return nil, err
	}
	return ring, nil
}

func (p *privateKeyRingScheme) counters(issuerid IssuerIdentifier) (i []uint, err error) {
	scheme := p.conf.SchemeManagers[issuerid.SchemeManagerIdentifier()]
	return matchKeyPattern(filepath.Join(scheme.path(), issuerid.Name(), "PrivateKeys", "*"))
}

func (p *privateKeyRingScheme) Get(id IssuerIdentifier, counter uint) (*gabi.PrivateKey, error) {
	schemeID := id.SchemeManagerIdentifier()
	scheme := p.conf.SchemeManagers[schemeID]
	if scheme == nil {
		return nil, errors.Errorf("Private key of issuer %s belongs to unknown scheme", id.String())
	}
	file := filepath.Join(scheme.path(), id.Name(), "PrivateKeys", strconv.FormatUint(uint64(counter), 10)+".xml")
	sk, err := gabi.NewPrivateKeyFromFile(file, scheme.Demo)
	if err != nil {
		return nil, err
	}
	if sk.Counter != counter {
		return nil, errors.Errorf("Private key %s of issuer %s has wrong <Counter>", file, id.String())
	}
	if err = validatePrivateKey(id, sk, p.conf); err != nil {
		return nil, err
	}
	return sk, nil
}

func (p *privateKeyRingScheme) Latest(id IssuerIdentifier) (*gabi.PrivateKey, error) {
	counters, err := p.counters(id)
	if err != nil {
		return nil, err
	}
	if len(counters) == 0 {
		return nil, ErrMissingPrivateKey
	}
	return p.Get(id, counters[len(counters)-1])
}

func (p *privateKeyRingScheme) Iterate(id IssuerIdentifier, f func(sk *gabi.PrivateKey) error) error {
	indices, err := p.counters(id)
	if err != nil {
		return err
	}
	for _, counter := range indices {
		sk, err := p.Get(id, counter)
		if err != nil {
			return err
		}
		if err = f(sk); err != nil {
			return err
		}
	}
	return nil
}

func (p *privateKeyRingMerge) Add(ring PrivateKeyRing) {
	p.rings = append(p.rings, ring)
}

func (p *privateKeyRingMerge) Get(id IssuerIdentifier, counter uint) (*gabi.PrivateKey, error) {
	for _, ring := range p.rings {
		sk, err := ring.Get(id, counter)
		if err == nil {
			return sk, nil
		}
		if !goerrors.Is(err, os.ErrNotExist) {
			return nil, err
		}
	}
	return nil, ErrMissingPrivateKey
}

func (p *privateKeyRingMerge) Latest(id IssuerIdentifier) (*gabi.PrivateKey, error) {
	var sk *gabi.PrivateKey
	for _, ring := range p.rings {
		s, err := ring.Latest(id)
		if err != nil && !goerrors.Is(err, os.ErrNotExist) {
			return nil, err
		}
		if s != nil && (sk == nil || s.Counter > sk.Counter) {
			sk = s
		}
	}
	if sk == nil {
		return nil, ErrMissingPrivateKey
	}
	return sk, nil
}

func (p *privateKeyRingMerge) Iterate(id IssuerIdentifier, f func(sk *gabi.PrivateKey) error) error {
	for _, ring := range p.rings {
		if err := ring.Iterate(id, f); err != nil {
			return err
		}
	}
	return nil
}

func validatePrivateKey(issuerid IssuerIdentifier, sk *gabi.PrivateKey, conf *Configuration) error {
	if _, ok := conf.Issuers[issuerid]; !ok {
		return errors.Errorf("Private key %d of issuer %s belongs to an unknown issuer", sk.Counter, issuerid.String())
	}
	pk, err := conf.PublicKey(issuerid, sk.Counter)
	if err != nil {
		return err
	}
	if pk == nil {
		return errors.Errorf("Private key %d of issuer %s has no corresponding public key", sk.Counter, issuerid.String())
	}
	if new(big.Int).Mul(sk.P, sk.Q).Cmp(pk.N) != 0 {
		return errors.Errorf("Private key %d of issuer %s does not belong to corresponding public key", sk.Counter, issuerid.String())
	}
	if sk.RevocationSupported() != pk.RevocationSupported() {
		msg := fmt.Sprintf("revocation support of private key %d of issuer %s is not consistent with corresponding public key", sk.Counter, issuerid.String())
		if conf.SchemeManagers[issuerid.SchemeManagerIdentifier()].Demo {
			Logger.Warn(msg)
		} else {
			return errors.Errorf(msg)
		}
	}
	return nil
}

func validatePrivateKeyRing(ring PrivateKeyRing, conf *Configuration) error {
	for issuerid := range conf.Issuers {
		err := ring.Iterate(issuerid, func(sk *gabi.PrivateKey) error {
			return validatePrivateKey(issuerid, sk, conf)
		})
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package view

import (
	"github.com/ungerik/go-start/debug"
	"github.com/ungerik/go-start/model"
)

const EditFormSliceTableScript = `
// function swapAttribs(a, b, attr) {
// 	var x = a.attr(attr);
// 	var y = b.attr(attr);
// 	a.attr(attr, y);
// 	b.attr(attr, x);
// }

function swapValues(a, b) {
	var x = a.val();
	var y = b.val();
	a.val(y);
	b.val(x);
}

function swapChecked(a, b) {
	var x = a.prop("checked");
	var y = b.prop("checked");
	a.prop("checked", y);
	b.prop("checked", x);
}

function swapRowValues(tr0, tr1) {
	var inputs0 = tr0.find("td > :input").not(":button");
	var inputs1 = tr1.find("td > :input").not(":button");
	for (i=0; i < inputs0.length; i++) {
		swapValues(inputs0.eq(i), inputs1.eq(i));
	}
	inputs0 = tr0.find("td > :checkbox");
	inputs1 = tr1.find("td > :checkbox");
	for (i=0; i < inputs0.length; i++) {
		swapChecked(inputs0.eq(i), inputs1.eq(i));
	}
}

function removeRow(button) {
	if (confirm("Are you sure you want to delete this row?")) {
		var tr = jQuery(button).parents("tr");

		// Swap all values with following rows to move the values of the
		// row to be deleted to the last row and everything else one row up
		var rows = tr.add(tr.nextAll());
		trs.each(function(index) {
			if (index == 0) return;
			swapRowValues(rows.eq(index-1), rows.eq(index));
		});

		var lastRowButtons = rows.last();
		var lastButOneRowButtons = lastRowButtons.prev();
		lastRowButtons = lastRowButtons.find("td:last");
		lastButOneRowButtons = lastButOneRowButtons.find("td:last");

		lastButOneRowButtons.empty();
		lastButOneRowButtons.append(lastRowButtons.children());

		// Delete the last row
		rows.last().remove();
	}
}

function addRow(button) {
	var tr0 = jQuery(button).parents("tr");
	var tr1 = tr0.clone();

	// Change the buttons of the old row
	var lastButton = tr0.find("td:last > button:last");
	var downButton = lastButton.prev();
	lastButton.attr("onclick", "removeRow(this);").text("X");
	downButton.removeProp("disabled");

	// Change the up button of the new row (could be disabled if first row)
	tr1.find("td:last > button:first").removeProp("disabled");

	// Set correct class for new row
	var numRows = tr0.prevAll().length + 1;
	var evenOdd = (numRows % 2 == 0) ? " even" : " odd";
	tr1.attr("class", "row"+numRows+evenOdd);

	// Correct name attributes of the new row's input elements
	var oldIndex = "."+(numRows-2)+".";
	var newIndex = "."+(numRows-1)+".";
	tr1.find("td > :input").not(":button").each(function(index) {
		var i = this.name.lastIndexOf(oldIndex);
		this.name = this.name.slice(0,i)+newIndex+this.name.slice(i+oldIndex.length);
	});

	tr1.insertAfter(tr0);

}

function moveRowUp(button) {
	var tr1 = jQuery(button).parents("tr");
	var tr0 = tr1.prev();
	swapRowValues(tr0, tr1);
}

function moveRowDown(button) {
	var tr0 = jQuery(button).parents("tr");
	var tr1 = tr0.next();
	swapRowValues(tr0, tr1);
}
`

/*
StandardFormLayout.

CSS needed for StandardFormLayout:

	form label:after {
		content: ":";
	}

	form input[type=checkbox] + label:after {
		content: "";
	}

Additional CSS for labels above input fields (except checkboxes):

	form label {
		display: block;
	}

	form input[type=checkbox] + label {
		display: inline;
	}

DIV classes for coloring:

	form .required {}
	form .error {}
	form .success {}

*/
type StandardFormLayout struct {
}

func (self *StandardFormLayout) BeginFormContent(form *Form, context *Context, formFields Views) Views {
	return formFields
}

func (self *StandardFormLayout) SubmitSuccess(message string, form *Form, context *Context, formFields Views) Views {
	return append(formFields, form.GetFieldFactory().NewSuccessMessage(message, form))
}

func (self *StandardFormLayout) SubmitError(message string, form *Form, context *Context, formFields Views) Views {
	return append(formFields, form.GetFieldFactory().NewGeneralErrorMessage(message, form))
}

func (self *StandardFormLayout) EndFormContent(fieldValidationErrs, generalValidationErrs []error, form *Form, context *Context, formFields Views) Views {
	fieldFactory := form.GetFieldFactory()
	for _, err := range generalValidationErrs {
		formFields = append(formFields, fieldFactory.NewGeneralErrorMessage(err.Error(), form))
		formFields = append(Views{fieldFactory.NewGeneralErrorMessage(err.Error(), form)}, formFields...)
	}
	formId := &HiddenInput{Name: FormIDName, Value: form.FormID}
	submitButton := fieldFactory.NewSubmitButton(form.GetSubmitButtonText(), form.SubmitButtonConfirm, form)
	return append(formFields, formId, submitButton)
}

func (self *StandardFormLayout) BeginStruct(strct *model.MetaData, form *Form, context *Context, formFields Views) Views {
	return formFields
}

func (self *StandardFormLayout) StructField(field *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	fieldFactory := form.GetFieldFactory()
	if !fieldFactory.CanCreateInput(field, form) || form.IsFieldExcluded(field) {
		return formFields
	}

	grandParent := field.Parent.Parent
	if grandParent != nil && (grandParent.Kind == model.ArrayKind || grandParent.Kind == model.SliceKind) {
		if form.IsFieldExcluded(grandParent) {
			return formFields
		}
		// We expect a Table as last form field.
		// If it doesn't exist yet because this is the first visible
		// struct field in the first array field, then create it
		var table *Table
		if len(formFields) > 0 {
			table, _ = formFields[len(formFields)-1].(*Table)
		}
		if table == nil {
			// First struct field of first array field, create table and table model
			table = &Table{
				Caption:   form.FieldLabel(grandParent),
				HeaderRow: true,
				Model:     ViewsTableModel{Views{}},
			}
			table.Init(table) // get an ID now
			formFields = append(formFields, table)
			// Add script for manipulating table rows
			context.AddScript(EditFormSliceTableScript, 0)
		}
		tableModel := table.Model.(ViewsTableModel)
		if field.Parent.Index == 0 {
			// If first array field, add label to table header
			tableModel[0] = append(tableModel[0], Escape(form.DirectFieldLabel(field)))
		}
		if tableModel.Rows()-1 == field.Parent.Index {
			// Create row in table model for this array field
			tableModel = append(tableModel, Views{})
			table.Model = tableModel
		}
		// Append form field in last row for this struct field
		row := &tableModel[tableModel.Rows()-1]
		*row = append(*row, fieldFactory.NewInput(false, field, form))

		return formFields
	}

	if form.IsFieldHidden(field) {
		return append(formFields, fieldFactory.NewHiddenInput(field, form))
	}
	var formField View = fieldFactory.NewInput(true, field, form)
	if validationErr != nil {
		formField = Views{formField, fieldFactory.NewFieldErrorMessage(validationErr.Error(), field, form)}
	}
	return append(formFields, DIV(Config.Form.StandardFormLayoutDivClass, formField))
}

func (self *StandardFormLayout) EndStruct(strct *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	return formFields
}

func (self *StandardFormLayout) BeginArray(array *model.MetaData, form *Form, context *Context, formFields Views) Views {
	debug.Nop()
	return formFields
}

func (self *StandardFormLayout) ArrayField(field *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	return self.StructField(field, validationErr, form, context, formFields) // todo replace
	return formFields
}

func (self *StandardFormLayout) EndArray(array *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	return formFields
}

func (self *StandardFormLayout) BeginSlice(slice *model.MetaData, form *Form, context *Context, formFields Views) Views {
	return formFields
}

func (self *StandardFormLayout) SliceField(field *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	return self.StructField(field, validationErr, form, context, formFields) // todo replace
	return formFields
}

func (self *StandardFormLayout) EndSlice(slice *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	if len(formFields) > 0 {
		if table, ok := formFields[len(formFields)-1].(*Table); ok {
			tableModel := table.Model.(ViewsTableModel)
			tableModel[0] = append(tableModel[0], HTML("Actions"))
			rows := tableModel.Rows()
			for i := 1; i < rows; i++ {
				firstRow := (i == 1)
				lastRow := (i == rows-1)
				tableModel[i] = append(
					tableModel[i],
					Views{
						HTML("  "),
						&Button{
							Content:  HTML("↑"),
							Disabled: firstRow,
							OnClick:  "moveRowUp(this);",
						},
						&Button{
							Content:  HTML("↓"),
							Disabled: lastRow,
							OnClick:  "moveRowDown(this);",
						},
						&If{
							Condition: lastRow,
							Content: &Button{
								Content: HTML("+"),
								OnClick: "addRow(this);",
							},
							ElseContent: &Button{
								Content: HTML("X"),
								OnClick: "removeRow(this)",
							},
						},
					},
				)
			}
		}
	}
	return formFields
}

func (self *StandardFormLayout) fieldNeedsLabel(field *model.MetaData) bool {
	switch field.Value.Addr().Interface().(type) {
	case *model.Bool:
		return false
	}
	return true
}
<commit_msg>form slice table buttons work now<commit_after>package view

import (
	"github.com/ungerik/go-start/debug"
	"github.com/ungerik/go-start/model"
)

const EditFormSliceTableScript = `
// function swapAttribs(a, b, attr) {
// 	var x = a.attr(attr);
// 	var y = b.attr(attr);
// 	a.attr(attr, y);
// 	b.attr(attr, x);
// }
// function swapChildren(a, b) {
// 	var x = a.children().detach();
// 	var y = b.children().detach();
// 	x.appendTo(b);
// 	y.appendTo(a);
// }

function swapValues(a, b) {
	var x = a.val();
	var y = b.val();
	a.val(y);
	b.val(x);
}

function swapChecked(a, b) {
	var x = a.prop("checked");
	var y = b.prop("checked");
	a.prop("checked", y);
	b.prop("checked", x);
}

function swapRowValues(tr0, tr1) {
	var inputs0 = tr0.find("td > :input").not(":button");
	var inputs1 = tr1.find("td > :input").not(":button");
	for (i=0; i < inputs0.length; i++) {
		swapValues(inputs0.eq(i), inputs1.eq(i));
	}
	inputs0 = tr0.find("td > :checkbox");
	inputs1 = tr1.find("td > :checkbox");
	for (i=0; i < inputs0.length; i++) {
		swapChecked(inputs0.eq(i), inputs1.eq(i));
	}
}

function resetButtons(table) {
	var rows = table.find("tr");
	rows.each(function(row) {
		var firstRow = (row == 1); // ignore header row
		var lastRow = (row == rows.length-1);
		var buttons = jQuery(this).find("td:last > :button");
		buttons.eq(0).prop("disabled", firstRow);
		buttons.eq(1).prop("disabled", lastRow);
		if (lastRow) {
			buttons.eq(2).attr("onclick", "addRow(this);").text("+");
		} else {
			buttons.eq(2).attr("onclick", "removeRow(this);").text("X");
		}
	});
}

function removeRow(button) {
	if (confirm("Are you sure you want to delete this row?")) {
		var tr = jQuery(button).parents("tr");
		var table = tr.parents("table");

		// Swap all values with following rows to move the values of the
		// row to be deleted to the last row and everything else one row up
		var rows = tr.add(tr.nextAll());
		rows.each(function(i) {
			if (i == 0) return;
			swapRowValues(rows.eq(i-1), rows.eq(i));
		});

		rows.last().remove();

		resetButtons(table);
	}
}

function addRow(button) {
	var tr0 = jQuery(button).parents("tr");
	var tr1 = tr0.clone();
	var table = tr0.parents("table");

	// Set correct class for new row
	var numRows = tr0.prevAll().length + 1;
	var evenOdd = (numRows % 2 == 0) ? " even" : " odd";
	tr1.attr("class", "row"+numRows+evenOdd);

	// Correct name attributes of the new row's input elements
	var oldIndex = "."+(numRows-2)+".";
	var newIndex = "."+(numRows-1)+".";
	tr1.find("td > :input").not(":button").each(function(index) {
		var i = this.name.lastIndexOf(oldIndex);
		this.name = this.name.slice(0,i)+newIndex+this.name.slice(i+oldIndex.length);
	});

	tr1.insertAfter(tr0);

	resetButtons(table);
}

function moveRowUp(button) {
	var tr1 = jQuery(button).parents("tr");
	var tr0 = tr1.prev();
	swapRowValues(tr0, tr1);
}

function moveRowDown(button) {
	var tr0 = jQuery(button).parents("tr");
	var tr1 = tr0.next();
	swapRowValues(tr0, tr1);
}
`

/*
StandardFormLayout.

CSS needed for StandardFormLayout:

	form label:after {
		content: ":";
	}

	form input[type=checkbox] + label:after {
		content: "";
	}

Additional CSS for labels above input fields (except checkboxes):

	form label {
		display: block;
	}

	form input[type=checkbox] + label {
		display: inline;
	}

DIV classes for coloring:

	form .required {}
	form .error {}
	form .success {}

*/
type StandardFormLayout struct {
}

func (self *StandardFormLayout) BeginFormContent(form *Form, context *Context, formFields Views) Views {
	return formFields
}

func (self *StandardFormLayout) SubmitSuccess(message string, form *Form, context *Context, formFields Views) Views {
	return append(formFields, form.GetFieldFactory().NewSuccessMessage(message, form))
}

func (self *StandardFormLayout) SubmitError(message string, form *Form, context *Context, formFields Views) Views {
	return append(formFields, form.GetFieldFactory().NewGeneralErrorMessage(message, form))
}

func (self *StandardFormLayout) EndFormContent(fieldValidationErrs, generalValidationErrs []error, form *Form, context *Context, formFields Views) Views {
	fieldFactory := form.GetFieldFactory()
	for _, err := range generalValidationErrs {
		formFields = append(formFields, fieldFactory.NewGeneralErrorMessage(err.Error(), form))
		formFields = append(Views{fieldFactory.NewGeneralErrorMessage(err.Error(), form)}, formFields...)
	}
	formId := &HiddenInput{Name: FormIDName, Value: form.FormID}
	submitButton := fieldFactory.NewSubmitButton(form.GetSubmitButtonText(), form.SubmitButtonConfirm, form)
	return append(formFields, formId, submitButton)
}

func (self *StandardFormLayout) BeginStruct(strct *model.MetaData, form *Form, context *Context, formFields Views) Views {
	return formFields
}

func (self *StandardFormLayout) StructField(field *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	fieldFactory := form.GetFieldFactory()
	if !fieldFactory.CanCreateInput(field, form) || form.IsFieldExcluded(field) {
		return formFields
	}

	grandParent := field.Parent.Parent
	if grandParent != nil && (grandParent.Kind == model.ArrayKind || grandParent.Kind == model.SliceKind) {
		if form.IsFieldExcluded(grandParent) {
			return formFields
		}
		// We expect a Table as last form field.
		// If it doesn't exist yet because this is the first visible
		// struct field in the first array field, then create it
		var table *Table
		if len(formFields) > 0 {
			table, _ = formFields[len(formFields)-1].(*Table)
		}
		if table == nil {
			// First struct field of first array field, create table and table model
			table = &Table{
				Caption:   form.FieldLabel(grandParent),
				HeaderRow: true,
				Model:     ViewsTableModel{Views{}},
			}
			table.Init(table) // get an ID now
			formFields = append(formFields, table)
			// Add script for manipulating table rows
			context.AddScript(EditFormSliceTableScript, 0)
		}
		tableModel := table.Model.(ViewsTableModel)
		if field.Parent.Index == 0 {
			// If first array field, add label to table header
			tableModel[0] = append(tableModel[0], Escape(form.DirectFieldLabel(field)))
		}
		if tableModel.Rows()-1 == field.Parent.Index {
			// Create row in table model for this array field
			tableModel = append(tableModel, Views{})
			table.Model = tableModel
		}
		// Append form field in last row for this struct field
		row := &tableModel[tableModel.Rows()-1]
		*row = append(*row, fieldFactory.NewInput(false, field, form))

		return formFields
	}

	if form.IsFieldHidden(field) {
		return append(formFields, fieldFactory.NewHiddenInput(field, form))
	}
	var formField View = fieldFactory.NewInput(true, field, form)
	if validationErr != nil {
		formField = Views{formField, fieldFactory.NewFieldErrorMessage(validationErr.Error(), field, form)}
	}
	return append(formFields, DIV(Config.Form.StandardFormLayoutDivClass, formField))
}

func (self *StandardFormLayout) EndStruct(strct *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	return formFields
}

func (self *StandardFormLayout) BeginArray(array *model.MetaData, form *Form, context *Context, formFields Views) Views {
	debug.Nop()
	return formFields
}

func (self *StandardFormLayout) ArrayField(field *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	return self.StructField(field, validationErr, form, context, formFields) // todo replace
	return formFields
}

func (self *StandardFormLayout) EndArray(array *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	return formFields
}

func (self *StandardFormLayout) BeginSlice(slice *model.MetaData, form *Form, context *Context, formFields Views) Views {
	return formFields
}

func (self *StandardFormLayout) SliceField(field *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	return self.StructField(field, validationErr, form, context, formFields) // todo replace
	return formFields
}

func (self *StandardFormLayout) EndSlice(slice *model.MetaData, validationErr error, form *Form, context *Context, formFields Views) Views {
	if len(formFields) > 0 {
		if table, ok := formFields[len(formFields)-1].(*Table); ok {
			tableModel := table.Model.(ViewsTableModel)
			tableModel[0] = append(tableModel[0], HTML("Actions"))
			rows := tableModel.Rows()
			for i := 1; i < rows; i++ {
				firstRow := (i == 1)
				lastRow := (i == rows-1)
				tableModel[i] = append(
					tableModel[i],
					Views{
						HTML("  "),
						&Button{
							Content:  HTML("↑"),
							Disabled: firstRow,
							OnClick:  "moveRowUp(this);",
						},
						&Button{
							Content:  HTML("↓"),
							Disabled: lastRow,
							OnClick:  "moveRowDown(this);",
						},
						&If{
							Condition: lastRow,
							Content: &Button{
								Content: HTML("+"),
								OnClick: "addRow(this);",
							},
							ElseContent: &Button{
								Content: HTML("X"),
								OnClick: "removeRow(this)",
							},
						},
					},
				)
			}
		}
	}
	return formFields
}

func (self *StandardFormLayout) fieldNeedsLabel(field *model.MetaData) bool {
	switch field.Value.Addr().Interface().(type) {
	case *model.Bool:
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package duckduckgo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"regexp"

	"github.com/PuerkitoBio/goquery"
)

type Session struct {
	cli http.Client
}

type WebResult struct {
	Url string
}

type ImageResult struct {
	Url string
}

type VideoResult struct {
	Id string
}

type ImageType string

const (
	ImageType_Any         ImageType = ""
	ImageType_Animated    ImageType = "type:photo-animatedgif"
	ImageType_Photo       ImageType = "type:photo-photo"
	ImageType_Clipart     ImageType = "type:photo-clipart"
	ImageType_Transparent ImageType = "type:photo-transparent"
)

var reVqd = regexp.MustCompile("vqd='([^']+)'")

func (sess *Session) Init() error {
	jar, err := cookiejar.New(nil)
	if err != nil {
		return err
	}
	sess.cli.Jar = jar
	return nil
}

func (sess *Session) IsInitialized() bool {
	return sess.cli.Jar != nil
}

func (sess *Session) Web(query string, offset uint) ([]WebResult, error) {
	fd := url.Values{
		"q":  []string{query},
		"b":  []string{},
		"kl": []string{"us-en"},
	}.Encode()
	req, err := sess.newRequest("POST", "https://duckduckgo.com/html/",
		bytes.NewBufferString(fd))
	if err != nil {
		return nil, err
	}
	req.Header.Set("content-type", "application/x-www-form-urlencoded")
	res, err := sess.cli.Do(req)
	if err != nil {
		return nil, err
	}
	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		return nil, err
	}
	results := make([]WebResult, 0, 16)
	doc.Find(".result__a").Each(func(i int, s *goquery.Selection) {
		if a, ok := s.Attr("href"); ok {
			results = append(results, WebResult{Url: a})
		}
	})
	return results, nil
}

func (sess *Session) Images(query string, safe bool,
	typ ImageType, offset uint) ([]ImageResult, error) {
	var vqd string
	{
		params := url.Values{
			"q":   []string{query},
			"iax": []string{"images"},
			"ia":  []string{"images"},
		}
		if typ != ImageType_Any {
			params.Set("iaf", string(typ))
		}
		u := url.URL{
			Scheme:   "https",
			Host:     "duckduckgo.com",
			Path:     "/",
			RawQuery: params.Encode(),
		}
		res, err := sess.request("GET", u.String(), nil)
		if err != nil {
			return nil, err
		}
		source, err := ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err != nil {
			return nil, err
		}
		m := reVqd.FindStringSubmatch(string(source))
		if m == nil {
			return nil, errors.New("invalid response")
		}
		vqd = m[1]
	}
	var results struct {
		Results []struct {
			Image string `json:"image"`
		} `json:"results"`
	}
	{
		params := url.Values{
			"l":   []string{"us-en"},
			"o":   []string{"json"},
			"q":   []string{query},
			"vqd": []string{vqd},
			"f":   []string{},
			"s":   []string{fmt.Sprint(offset)},
		}
		if typ != ImageType_Any {
			params.Set("f", ","+string(typ)+",,")
		}
		if !safe {
			params.Set("p", "-1")
		}
		u := url.URL{
			Scheme:   "https",
			Host:     "duckduckgo.com",
			Path:     "/i.js",
			RawQuery: params.Encode(),
		}
		res, err := sess.request("GET", u.String(), nil)
		if err != nil {
			return nil, err
		}
		if err := json.NewDecoder(res.Body).Decode(&results); err != nil {
			res.Body.Close()
			return nil, err
		}
		res.Body.Close()
	}
	if len(results.Results) == 0 {
		return []ImageResult{}, nil
	}
	images := make([]ImageResult, len(results.Results))
	for i := 0; i < len(results.Results); i++ {
		images[i].Url = results.Results[i].Image
	}
	return images, nil
}

func (sess *Session) Videos(query string, offset uint) ([]VideoResult, error) {
	var vqd string
	{
		u := url.URL{
			Scheme: "https",
			Host:   "duckduckgo.com",
			Path:   "/",
			RawQuery: url.Values{
				"q":   []string{query},
				"iax": []string{"1"},
				"ia":  []string{"videos"},
				"t":   []string{"h_"},
			}.Encode(),
		}
		res, err := sess.request("GET", u.String(), nil)
		if err != nil {
			return nil, err
		}
		source, err := ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err != nil {
			return nil, err
		}
		m := reVqd.FindStringSubmatch(string(source))
		if m == nil {
			return nil, errors.New("invalid response")
		}
		vqd = m[1]
	}
	var results struct {
		Results []struct {
			Provider string `json:"provider"`
			ID       string `json:"id"`
		} `json:"results"`
	}
	{
		u := url.URL{
			Scheme: "https",
			Host:   "duckduckgo.com",
			Path:   "/v.js",
			RawQuery: url.Values{
				"o":      []string{"json"},
				"strict": []string{"1"},
				"q":      []string{query},
				"vqd":    []string{vqd},
				"s":      []string{fmt.Sprint(offset)},
			}.Encode(),
		}
		res, err := sess.request("GET", u.String(), nil)
		if err != nil {
			return nil, err
		}
		if err := json.NewDecoder(res.Body).Decode(&results); err != nil {
			res.Body.Close()
			return nil, err
		}
		res.Body.Close()
	}
	ids := make([]VideoResult, 0, len(results.Results))
	for _, v := range results.Results {
		if v.Provider == "YouTube" {
			ids = append(ids, VideoResult{Id: v.ID})
		}
	}
	if len(ids) == 0 {
		return []VideoResult{}, nil
	}
	return ids, nil
}

func (sess *Session) request(method, url string, body io.Reader) (*http.Response, error) {
	req, err := sess.newRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	return sess.cli.Do(req)
}

func (sess *Session) newRequest(method, url string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("user-agent",
		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36")
	return req, nil
}
<commit_msg>http client timeout added<commit_after>package duckduckgo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"regexp"

	"github.com/PuerkitoBio/goquery"
)

type Session struct {
	cli http.Client
}

type WebResult struct {
	Url string
}

type ImageResult struct {
	Url string
}

type VideoResult struct {
	Id string
}

type ImageType string

const (
	ImageType_Any         ImageType = ""
	ImageType_Animated    ImageType = "type:photo-animatedgif"
	ImageType_Photo       ImageType = "type:photo-photo"
	ImageType_Clipart     ImageType = "type:photo-clipart"
	ImageType_Transparent ImageType = "type:photo-transparent"
)

var reVqd = regexp.MustCompile("vqd='([^']+)'")

func (sess *Session) Init() error {
	jar, err := cookiejar.New(nil)
	if err != nil {
		return err
	}
	sess.cli.Jar = jar
	sess.cli.Timeout = time.Second * 10
	return nil
}

func (sess *Session) IsInitialized() bool {
	return sess.cli.Jar != nil
}

func (sess *Session) Web(query string, offset uint) ([]WebResult, error) {
	fd := url.Values{
		"q":  []string{query},
		"b":  []string{},
		"kl": []string{"us-en"},
	}.Encode()
	req, err := sess.newRequest("POST", "https://duckduckgo.com/html/",
		bytes.NewBufferString(fd))
	if err != nil {
		return nil, err
	}
	req.Header.Set("content-type", "application/x-www-form-urlencoded")
	res, err := sess.cli.Do(req)
	if err != nil {
		return nil, err
	}
	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		return nil, err
	}
	results := make([]WebResult, 0, 16)
	doc.Find(".result__a").Each(func(i int, s *goquery.Selection) {
		if a, ok := s.Attr("href"); ok {
			results = append(results, WebResult{Url: a})
		}
	})
	return results, nil
}

func (sess *Session) Images(query string, safe bool,
	typ ImageType, offset uint) ([]ImageResult, error) {
	var vqd string
	{
		params := url.Values{
			"q":   []string{query},
			"iax": []string{"images"},
			"ia":  []string{"images"},
		}
		if typ != ImageType_Any {
			params.Set("iaf", string(typ))
		}
		u := url.URL{
			Scheme:   "https",
			Host:     "duckduckgo.com",
			Path:     "/",
			RawQuery: params.Encode(),
		}
		res, err := sess.request("GET", u.String(), nil)
		if err != nil {
			return nil, err
		}
		source, err := ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err != nil {
			return nil, err
		}
		m := reVqd.FindStringSubmatch(string(source))
		if m == nil {
			return nil, errors.New("invalid response")
		}
		vqd = m[1]
	}
	var results struct {
		Results []struct {
			Image string `json:"image"`
		} `json:"results"`
	}
	{
		params := url.Values{
			"l":   []string{"us-en"},
			"o":   []string{"json"},
			"q":   []string{query},
			"vqd": []string{vqd},
			"f":   []string{},
			"s":   []string{fmt.Sprint(offset)},
		}
		if typ != ImageType_Any {
			params.Set("f", ","+string(typ)+",,")
		}
		if !safe {
			params.Set("p", "-1")
		}
		u := url.URL{
			Scheme:   "https",
			Host:     "duckduckgo.com",
			Path:     "/i.js",
			RawQuery: params.Encode(),
		}
		res, err := sess.request("GET", u.String(), nil)
		if err != nil {
			return nil, err
		}
		if err := json.NewDecoder(res.Body).Decode(&results); err != nil {
			res.Body.Close()
			return nil, err
		}
		res.Body.Close()
	}
	if len(results.Results) == 0 {
		return []ImageResult{}, nil
	}
	images := make([]ImageResult, len(results.Results))
	for i := 0; i < len(results.Results); i++ {
		images[i].Url = results.Results[i].Image
	}
	return images, nil
}

func (sess *Session) Videos(query string, offset uint) ([]VideoResult, error) {
	var vqd string
	{
		u := url.URL{
			Scheme: "https",
			Host:   "duckduckgo.com",
			Path:   "/",
			RawQuery: url.Values{
				"q":   []string{query},
				"iax": []string{"1"},
				"ia":  []string{"videos"},
				"t":   []string{"h_"},
			}.Encode(),
		}
		res, err := sess.request("GET", u.String(), nil)
		if err != nil {
			return nil, err
		}
		source, err := ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err != nil {
			return nil, err
		}
		m := reVqd.FindStringSubmatch(string(source))
		if m == nil {
			return nil, errors.New("invalid response")
		}
		vqd = m[1]
	}
	var results struct {
		Results []struct {
			Provider string `json:"provider"`
			ID       string `json:"id"`
		} `json:"results"`
	}
	{
		u := url.URL{
			Scheme: "https",
			Host:   "duckduckgo.com",
			Path:   "/v.js",
			RawQuery: url.Values{
				"o":      []string{"json"},
				"strict": []string{"1"},
				"q":      []string{query},
				"vqd":    []string{vqd},
				"s":      []string{fmt.Sprint(offset)},
			}.Encode(),
		}
		res, err := sess.request("GET", u.String(), nil)
		if err != nil {
			return nil, err
		}
		if err := json.NewDecoder(res.Body).Decode(&results); err != nil {
			res.Body.Close()
			return nil, err
		}
		res.Body.Close()
	}
	ids := make([]VideoResult, 0, len(results.Results))
	for _, v := range results.Results {
		if v.Provider == "YouTube" {
			ids = append(ids, VideoResult{Id: v.ID})
		}
	}
	if len(ids) == 0 {
		return []VideoResult{}, nil
	}
	return ids, nil
}

func (sess *Session) request(method, url string, body io.Reader) (*http.Response, error) {
	req, err := sess.newRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	return sess.cli.Do(req)
}

func (sess *Session) newRequest(method, url string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("user-agent",
		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36")
	return req, nil
}
<|endoftext|>


<commit_before>package blockchain

import (
	crand "crypto/rand"
	"crypto/sha256"
	"math/big"
	mrand "math/rand"

	c "github.com/ubclaunchpad/cumulus/common/constants"
	"github.com/ubclaunchpad/cumulus/common/util"
)

// NewTestHash produces a hash.
func NewTestHash() Hash {
	message := make([]byte, 256)
	crand.Read(message)
	return sha256.Sum256(message)
}

// NewTestTxHashPointer produces transaction hash pointer.
func NewTestTxHashPointer() TxHashPointer {
	return TxHashPointer{
		BlockNumber: mrand.Uint32(),
		Hash:        NewTestHash(),
		Index:       mrand.Uint32(),
	}
}

// NewTestTxOutput random txn output.
func NewTestTxOutput() TxOutput {
	return TxOutput{
		Amount:    uint64(mrand.Int63()),
		Recipient: NewWallet().Public().Repr(),
	}
}

// NewTestTxBody random txn body.
func NewTestTxBody() TxBody {
	// Uniform distribution on [1, 4]
	nOutputs := mrand.Intn(4) + 1
	nInputs := mrand.Intn(4) + 1
	body := TxBody{
		Sender:  NewWallet().Public(),
		Inputs:  make([]TxHashPointer, nInputs),
		Outputs: make([]TxOutput, nOutputs),
	}
	for i := 0; i < nOutputs; i++ {
		body.Outputs[i] = NewTestTxOutput()
	}
	for i := 0; i < nInputs; i++ {
		body.Inputs[i] = NewTestTxHashPointer()
	}
	return body
}

// NewTestTransaction prodcues random txn.
func NewTestTransaction() *Transaction {
	sender := NewWallet()
	tbody := NewTestTxBody()
	t, _ := tbody.Sign(*sender, crand.Reader)
	return t
}

// NewTestBlockHeader prodcues random block header.
func NewTestBlockHeader() BlockHeader {
	return BlockHeader{
		BlockNumber: mrand.Uint32(),
		LastBlock:   NewTestHash(),
		Target:      NewValidTestTarget(),
		Time:        mrand.Uint32(),
		Nonce:       0,
	}
}

// NewTestBlock prodcues random block.
func NewTestBlock() *Block {
	// Uniform distribution on [500, 999]
	nTransactions := mrand.Intn(500) + 500
	b := Block{
		BlockHeader:  NewTestBlockHeader(),
		Transactions: make([]*Transaction, nTransactions),
	}
	for i := 0; i < nTransactions; i++ {
		b.Transactions[i] = NewTestTransaction()
	}
	return &b
}

// NewTestBlockChain produces random blockchain.
func NewTestBlockChain() *BlockChain {
	// Uniform distribution on [10, 50]
	nBlocks := mrand.Intn(40) + 10
	bc := BlockChain{Blocks: make([]*Block, nBlocks)}
	for i := 0; i < nBlocks; i++ {
		bc.Blocks[i] = NewTestBlock()
	}
	bc.Head = HashSum(bc.Blocks[nBlocks-1])
	return &bc
}

// NewTestInputBlock produces new block with given transactions.
func NewTestInputBlock(t []*Transaction) *Block {
	return &Block{
		BlockHeader: BlockHeader{
			BlockNumber: 0,
			LastBlock:   NewTestHash(),
			Target:      NewValidTestTarget(),
			Time:        util.UnixNow(),
			Nonce:       0,
		},
		Transactions: t,
	}
}

// NewTestOutputBlock produces new block with given transactions and given input
// block.
func NewTestOutputBlock(t []*Transaction, input *Block) *Block {
	return &Block{
		BlockHeader: BlockHeader{
			BlockNumber: input.BlockNumber + 1,
			LastBlock:   HashSum(input),
			Target:      NewValidTestTarget(),
			Time:        util.UnixNow(),
			Nonce:       0,
		},
		Transactions: t,
	}
}

// NewTestTransactionValue creates a new transaction with specific value a at
// index i in block number b.
func NewTestTransactionValue(s, r *Wallet, a uint64, i uint32, b uint32) (*Transaction, error) {
	tbody := TxBody{
		Sender:  s.Public(),
		Inputs:  make([]TxHashPointer, 1),
		Outputs: make([]TxOutput, 1),
	}
	tbody.Outputs[0] = TxOutput{
		Amount:    a,
		Recipient: r.Public().Repr(),
	}
	tbody.Inputs[0] = TxHashPointer{
		BlockNumber: b,
		Hash:        NewTestHash(),
		Index:       i,
	}
	return tbody.Sign(*s, crand.Reader)
}

// NewValidBlockChainFixture creates a valid blockchain of three blocks
// and returns the wallets involved in the transactions.
// The returning wallets will have balances of 3, 1, and 0 respectively.
func NewValidBlockChainFixture() (*BlockChain, map[string]*Wallet) {
	sender := NewWallet()
	alice := NewWallet()
	bob := NewWallet()

	// Cloud base txns for our blocks.
	cbA, _ := NewValidCloudBaseTestTransaction()
	cbB, _ := NewValidCloudBaseTestTransaction()
	cbC, _ := NewValidCloudBaseTestTransaction()

	// Transaction A is at index 1 in block 0 (sender awarded 4 coins).
	tA, _ := TxBody{
		Sender: sender.Public(),
		Inputs: []TxHashPointer{},
		Outputs: []TxOutput{
			TxOutput{
				Amount:    2,
				Recipient: sender.Public().Repr(),
			},
			TxOutput{
				Amount:    2,
				Recipient: sender.Public().Repr(),
			},
		},
	}.Sign(*sender, crand.Reader)

	block0 := Block{
		BlockHeader: BlockHeader{
			BlockNumber: 0,
			LastBlock:   NewTestHash(),
			Target:      NewValidTestTarget(),
			Time:        mrand.Uint32(),
			Nonce:       0,
		},
		// Block0 is a cb and a transaction.
		Transactions: []*Transaction{cbA, tA},
	}

	// Transaction B is at index 1 in block 1 (sender sends 3 coins to recipientA).
	tB, _ := TxBody{
		Sender: sender.Public(),
		Inputs: []TxHashPointer{
			TxHashPointer{
				// Reference block 0, index 0 for inputs.
				BlockNumber: 0,
				Index:       1, // Cloudbase will bump transactions forward.
				Hash:        HashSum(tA),
			},
		},
		// Send some outputs to recipientA, some back to self.
		Outputs: []TxOutput{
			TxOutput{
				Amount:    3,
				Recipient: alice.Public().Repr(),
			},
			TxOutput{
				Amount:    1,
				Recipient: sender.Public().Repr(),
			},
		},
	}.Sign(*sender, crand.Reader)

	// Block1 is a cb and a transaction.
	block1 := Block{
		BlockHeader: BlockHeader{
			BlockNumber: 1,
			LastBlock:   HashSum(block0),
			Target:      NewValidTestTarget(),
			Time:        mrand.Uint32(),
			Nonce:       0,
		},
		Transactions: []*Transaction{cbB, tB},
	}

	// Sender has 1 coin left to send to bob.
	tC, _ := TxBody{
		Sender: sender.Public(),
		Inputs: []TxHashPointer{
			TxHashPointer{
				// Again look at block 1.
				BlockNumber: 1,
				Index:       1, // skip cb
				Hash:        HashSum(tB),
			},
		},
		// One coin output to recipientB.
		Outputs: []TxOutput{
			TxOutput{
				Amount:    1,
				Recipient: bob.Public().Repr(),
			},
		},
	}.Sign(*sender, crand.Reader)

	// Block2 is a cb and a transaction.
	block2 := Block{
		BlockHeader: BlockHeader{
			BlockNumber: 2,
			LastBlock:   HashSum(block1),
			Target:      NewValidTestTarget(),
			Time:        mrand.Uint32(),
			Nonce:       0,
		},
		Transactions: []*Transaction{cbC, tC},
	}

	wallets := map[string]*Wallet{
		"alice":  alice,
		"bob":    bob,
		"sender": sender,
	}

	return &BlockChain{
		Blocks: []*Block{&block0, &block1, &block2},
		Head:   NewTestHash(),
	}, wallets // Wallet balances 3, 1, 0.
}

// NewValidTestChainAndBlock creates a valid BlockChain of 3 blocks,
// and a new block which is valid with respect to the blockchain.
func NewValidTestChainAndBlock() (*BlockChain, *Block) {
	bc, wallets := NewValidBlockChainFixture()

	// Alice wants to send 2 coins to bob and bob wants to send
	// his coin back to the sender.
	aliceToBob, _ := TxBody{
		Sender: wallets["alice"].Public(),
		Inputs: []TxHashPointer{
			TxHashPointer{
				// Block 1, transaction 1 is where this input comes from.
				BlockNumber: 1,
				Index:       1,
				Hash:        HashSum(bc.Blocks[1].Transactions[1]),
			},
		},
		// One output to bob, one back to alice.
		Outputs: []TxOutput{
			TxOutput{
				Amount:    2,
				Recipient: wallets["bob"].Public().Repr(),
			},
			TxOutput{
				Amount:    1,
				Recipient: wallets["alice"].Public().Repr(),
			},
		},
	}.Sign(*wallets["alice"], crand.Reader)

	bobToSender, _ := TxBody{
		Sender: wallets["bob"].Public(),
		Inputs: []TxHashPointer{
			TxHashPointer{
				// Block 2, transaction 1 is where this input comes from.
				BlockNumber: 2,
				Index:       1,
				Hash:        HashSum(bc.Blocks[2].Transactions[1]),
			},
		},
		// One output to sender.
		Outputs: []TxOutput{
			TxOutput{
				Amount:    1,
				Recipient: wallets["sender"].Public().Repr(),
			},
		},
	}.Sign(*wallets["bob"], crand.Reader)

	cb, _ := NewValidCloudBaseTestTransaction()

	blk := Block{
		BlockHeader: BlockHeader{
			BlockNumber: 2,
			LastBlock:   HashSum(bc.Blocks[2]),
			Target:      NewValidTestTarget(),
			Time:        mrand.Uint32(),
			Nonce:       0,
		},
		Transactions: []*Transaction{cb, aliceToBob, bobToSender},
	}

	return bc, &blk
}

// NewValidChainAndTxn creates a valid BlockChain of 3 blocks,
// and a Transaction that is valid with respect to the BlockChain.
func NewValidChainAndTxn() (*BlockChain, *Transaction) {
	bc, b := NewValidTestChainAndBlock()
	return bc, b.Transactions[1]
}

// NewValidTestTarget creates a new valid target that is a random value between the
// max and min difficulties
func NewValidTestTarget() Hash {
	return BigIntToHash(
		new(big.Int).Div(
			c.MaxTarget,
			c.MinTarget,
		),
	)
}

// NewValidCloudBaseTestTransaction returns a new valid CloudBase transaction and
// the address of the recipient of the transaction
func NewValidCloudBaseTestTransaction() (*Transaction, Address) {
	w := NewWallet()
	cbInput := TxHashPointer{
		BlockNumber: 0,
		Hash:        NilHash,
		Index:       0,
	}
	cbReward := TxOutput{
		Amount:    25,
		Recipient: w.Public().Repr(),
	}
	cbTxBody := TxBody{
		Sender:  NilAddr,
		Inputs:  []TxHashPointer{cbInput},
		Outputs: []TxOutput{cbReward},
	}
	cbTx := &Transaction{
		TxBody: cbTxBody,
		Sig:    NilSig,
	}
	return cbTx, w.Public()
}
<commit_msg>All tests pass for blocks<commit_after>package blockchain

import (
	crand "crypto/rand"
	"crypto/sha256"
	"math/big"
	mrand "math/rand"

	c "github.com/ubclaunchpad/cumulus/common/constants"
	"github.com/ubclaunchpad/cumulus/common/util"
)

// NewTestHash produces a hash.
func NewTestHash() Hash {
	message := make([]byte, 256)
	crand.Read(message)
	return sha256.Sum256(message)
}

// NewTestTxHashPointer produces transaction hash pointer.
func NewTestTxHashPointer() TxHashPointer {
	return TxHashPointer{
		BlockNumber: mrand.Uint32(),
		Hash:        NewTestHash(),
		Index:       mrand.Uint32(),
	}
}

// NewTestTxOutput random txn output.
func NewTestTxOutput() TxOutput {
	return TxOutput{
		Amount:    uint64(mrand.Int63()),
		Recipient: NewWallet().Public().Repr(),
	}
}

// NewTestTxBody random txn body.
func NewTestTxBody() TxBody {
	// Uniform distribution on [1, 4]
	nOutputs := mrand.Intn(4) + 1
	nInputs := mrand.Intn(4) + 1
	body := TxBody{
		Sender:  NewWallet().Public(),
		Inputs:  make([]TxHashPointer, nInputs),
		Outputs: make([]TxOutput, nOutputs),
	}
	for i := 0; i < nOutputs; i++ {
		body.Outputs[i] = NewTestTxOutput()
	}
	for i := 0; i < nInputs; i++ {
		body.Inputs[i] = NewTestTxHashPointer()
	}
	return body
}

// NewTestTransaction prodcues random txn.
func NewTestTransaction() *Transaction {
	sender := NewWallet()
	tbody := NewTestTxBody()
	t, _ := tbody.Sign(*sender, crand.Reader)
	return t
}

// NewTestBlockHeader prodcues random block header.
func NewTestBlockHeader() BlockHeader {
	return BlockHeader{
		BlockNumber: mrand.Uint32(),
		LastBlock:   NewTestHash(),
		Target:      NewValidTestTarget(),
		Time:        mrand.Uint32(),
		Nonce:       0,
	}
}

// NewTestBlock prodcues random block.
func NewTestBlock() *Block {
	// Uniform distribution on [500, 999]
	nTransactions := mrand.Intn(500) + 500
	b := Block{
		BlockHeader:  NewTestBlockHeader(),
		Transactions: make([]*Transaction, nTransactions),
	}
	for i := 0; i < nTransactions; i++ {
		b.Transactions[i] = NewTestTransaction()
	}
	return &b
}

// NewTestBlockChain produces random blockchain.
func NewTestBlockChain() *BlockChain {
	// Uniform distribution on [10, 50]
	nBlocks := mrand.Intn(40) + 10
	bc := BlockChain{Blocks: make([]*Block, nBlocks)}
	for i := 0; i < nBlocks; i++ {
		bc.Blocks[i] = NewTestBlock()
	}
	bc.Head = HashSum(bc.Blocks[nBlocks-1])
	return &bc
}

// NewTestInputBlock produces new block with given transactions.
func NewTestInputBlock(t []*Transaction) *Block {
	return &Block{
		BlockHeader: BlockHeader{
			BlockNumber: 0,
			LastBlock:   NewTestHash(),
			Target:      NewValidTestTarget(),
			Time:        util.UnixNow(),
			Nonce:       0,
		},
		Transactions: t,
	}
}

// NewTestOutputBlock produces new block with given transactions and given input
// block.
func NewTestOutputBlock(t []*Transaction, input *Block) *Block {
	return &Block{
		BlockHeader: BlockHeader{
			BlockNumber: input.BlockNumber + 1,
			LastBlock:   HashSum(input),
			Target:      NewValidTestTarget(),
			Time:        util.UnixNow(),
			Nonce:       0,
		},
		Transactions: t,
	}
}

// NewTestTransactionValue creates a new transaction with specific value a at
// index i in block number b.
func NewTestTransactionValue(s, r *Wallet, a uint64, i uint32, b uint32) (*Transaction, error) {
	tbody := TxBody{
		Sender:  s.Public(),
		Inputs:  make([]TxHashPointer, 1),
		Outputs: make([]TxOutput, 1),
	}
	tbody.Outputs[0] = TxOutput{
		Amount:    a,
		Recipient: r.Public().Repr(),
	}
	tbody.Inputs[0] = TxHashPointer{
		BlockNumber: b,
		Hash:        NewTestHash(),
		Index:       i,
	}
	return tbody.Sign(*s, crand.Reader)
}

// NewValidBlockChainFixture creates a valid blockchain of three blocks
// and returns the wallets involved in the transactions.
// The returning wallets will have balances of 3, 1, and 0 respectively.
func NewValidBlockChainFixture() (*BlockChain, map[string]*Wallet) {
	sender := NewWallet()
	alice := NewWallet()
	bob := NewWallet()

	// Cloud base txns for our blocks.
	cbA, _ := NewValidCloudBaseTestTransaction()
	cbB, _ := NewValidCloudBaseTestTransaction()
	cbC, _ := NewValidCloudBaseTestTransaction()

	// Transaction A is at index 1 in block 0 (sender awarded 4 coins).
	tA, _ := TxBody{
		Sender: sender.Public(),
		Inputs: []TxHashPointer{},
		Outputs: []TxOutput{
			TxOutput{
				Amount:    2,
				Recipient: sender.Public().Repr(),
			},
			TxOutput{
				Amount:    2,
				Recipient: sender.Public().Repr(),
			},
		},
	}.Sign(*sender, crand.Reader)

	block0 := Block{
		BlockHeader: BlockHeader{
			BlockNumber: 0,
			LastBlock:   NewTestHash(),
			Target:      NewValidTestTarget(),
			Time:        mrand.Uint32(),
			Nonce:       0,
		},
		// Block0 is a cb and a transaction.
		Transactions: []*Transaction{cbA, tA},
	}

	// Transaction B is at index 1 in block 1 (sender sends 3 coins to recipientA).
	tB, _ := TxBody{
		Sender: sender.Public(),
		Inputs: []TxHashPointer{
			TxHashPointer{
				// Reference block 0, index 0 for inputs.
				BlockNumber: 0,
				Index:       1, // Cloudbase will bump transactions forward.
				Hash:        HashSum(tA),
			},
		},
		// Send some outputs to recipientA, some back to self.
		Outputs: []TxOutput{
			TxOutput{
				Amount:    3,
				Recipient: alice.Public().Repr(),
			},
			TxOutput{
				Amount:    1,
				Recipient: sender.Public().Repr(),
			},
		},
	}.Sign(*sender, crand.Reader)

	// Block1 is a cb and a transaction.
	block1 := Block{
		BlockHeader: BlockHeader{
			BlockNumber: 1,
			LastBlock:   HashSum(block0),
			Target:      NewValidTestTarget(),
			Time:        mrand.Uint32(),
			Nonce:       0,
		},
		Transactions: []*Transaction{cbB, tB},
	}

	// Sender has 1 coin left to send to bob.
	tC, _ := TxBody{
		Sender: sender.Public(),
		Inputs: []TxHashPointer{
			TxHashPointer{
				// Again look at block 1.
				BlockNumber: 1,
				Index:       1, // skip cb
				Hash:        HashSum(tB),
			},
		},
		// One coin output to recipientB.
		Outputs: []TxOutput{
			TxOutput{
				Amount:    1,
				Recipient: bob.Public().Repr(),
			},
		},
	}.Sign(*sender, crand.Reader)

	// Block2 is a cb and a transaction.
	block2 := Block{
		BlockHeader: BlockHeader{
			BlockNumber: 2,
			LastBlock:   HashSum(block1),
			Target:      NewValidTestTarget(),
			Time:        mrand.Uint32(),
			Nonce:       0,
		},
		Transactions: []*Transaction{cbC, tC},
	}

	wallets := map[string]*Wallet{
		"alice":  alice,
		"bob":    bob,
		"sender": sender,
	}

	return &BlockChain{
		Blocks: []*Block{&block0, &block1, &block2},
		Head:   NewTestHash(),
	}, wallets // Wallet balances 3, 1, 0.
}

// NewValidTestChainAndBlock creates a valid BlockChain of 3 blocks,
// and a new block which is valid with respect to the blockchain.
func NewValidTestChainAndBlock() (*BlockChain, *Block) {
	bc, wallets := NewValidBlockChainFixture()

	// Alice wants to send 2 coins to bob and bob wants to send
	// his coin back to the sender.
	aliceToBob, _ := TxBody{
		Sender: wallets["alice"].Public(),
		Inputs: []TxHashPointer{
			TxHashPointer{
				// Block 1, transaction 1 is where this input comes from.
				BlockNumber: 1,
				Index:       1,
				Hash:        HashSum(bc.Blocks[1].Transactions[1]),
			},
		},
		// One output to bob, one back to alice.
		Outputs: []TxOutput{
			TxOutput{
				Amount:    2,
				Recipient: wallets["bob"].Public().Repr(),
			},
			TxOutput{
				Amount:    1,
				Recipient: wallets["alice"].Public().Repr(),
			},
		},
	}.Sign(*wallets["alice"], crand.Reader)

	bobToSender, _ := TxBody{
		Sender: wallets["bob"].Public(),
		Inputs: []TxHashPointer{
			TxHashPointer{
				// Block 2, transaction 1 is where this input comes from.
				BlockNumber: 2,
				Index:       1,
				Hash:        HashSum(bc.Blocks[2].Transactions[1]),
			},
		},
		// One output to sender.
		Outputs: []TxOutput{
			TxOutput{
				Amount:    1,
				Recipient: wallets["sender"].Public().Repr(),
			},
		},
	}.Sign(*wallets["bob"], crand.Reader)

	cb, _ := NewValidCloudBaseTestTransaction()

	blk := Block{
		BlockHeader: BlockHeader{
			BlockNumber: 3,
			LastBlock:   HashSum(bc.Blocks[2]),
			Target:      NewValidTestTarget(),
			Time:        mrand.Uint32(),
			Nonce:       0,
		},
		Transactions: []*Transaction{cb, aliceToBob, bobToSender},
	}

	return bc, &blk
}

// NewValidChainAndTxn creates a valid BlockChain of 3 blocks,
// and a Transaction that is valid with respect to the BlockChain.
func NewValidChainAndTxn() (*BlockChain, *Transaction) {
	bc, b := NewValidTestChainAndBlock()
	return bc, b.Transactions[1]
}

// NewValidTestTarget creates a new valid target that is a random value between the
// max and min difficulties
func NewValidTestTarget() Hash {
	return BigIntToHash(
		new(big.Int).Div(
			c.MaxTarget,
			c.MinTarget,
		),
	)
}

// NewValidCloudBaseTestTransaction returns a new valid CloudBase transaction and
// the address of the recipient of the transaction
func NewValidCloudBaseTestTransaction() (*Transaction, Address) {
	w := NewWallet()
	cbInput := TxHashPointer{
		BlockNumber: 0,
		Hash:        NilHash,
		Index:       0,
	}
	cbReward := TxOutput{
		Amount:    25,
		Recipient: w.Public().Repr(),
	}
	cbTxBody := TxBody{
		Sender:  NilAddr,
		Inputs:  []TxHashPointer{cbInput},
		Outputs: []TxOutput{cbReward},
	}
	cbTx := &Transaction{
		TxBody: cbTxBody,
		Sig:    NilSig,
	}
	return cbTx, w.Public()
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package amppackager

import (
	"bytes"
	"crypto/x509"
	"net/http"
	"net/url"
	"path"
	"time"

	"github.com/nyaxt/webpackage/go/signedexchange/certurl"
	"github.com/pkg/errors"
)

type CertCache struct {
	// TODO(twifkak): Support multiple certs.
	certName    string
	certMessage []byte
}

func NewCertCache(cert *x509.Certificate, pemContent []byte) (*CertCache, error) {
	this := new(CertCache)
	this.certName = CertName(cert)
	// TODO(twifkak): Refactor CertificateMessageFromPEM to be based on the x509.Certificate instead.
	var err error
	this.certMessage, err = certurl.CertificateMessageFromPEM(pemContent)
	if err != nil {
		return nil, errors.Wrap(err, "extracting certificate from CertFile")
	}
	return this, nil
}

func (this CertCache) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	println("path", req.URL.Path)
	if req.URL.Path == path.Join("/", CertURLPrefix, url.PathEscape(this.certName)) {
		// https://tools.ietf.org/html/draft-yasskin-httpbis-origin-signed-exchanges-impl-00#section-3.3
		resp.Header().Set("Content-Type", "application/octet-stream")
		resp.Header().Set("ETag", "\""+this.certName+"\"")
		// TODO(twifkak): Add cache headers.
		http.ServeContent(resp, req, "", time.Time{}, bytes.NewReader(this.certMessage))
	} else {
		http.NotFound(resp, req)
	}
}
<commit_msg>Add a 7-day cache lifetime for the certUrl.<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package amppackager

import (
	"bytes"
	"crypto/x509"
	"net/http"
	"net/url"
	"path"
	"time"

	"github.com/nyaxt/webpackage/go/signedexchange/certurl"
	"github.com/pkg/errors"
)

type CertCache struct {
	// TODO(twifkak): Support multiple certs.
	certName    string
	certMessage []byte
}

func NewCertCache(cert *x509.Certificate, pemContent []byte) (*CertCache, error) {
	this := new(CertCache)
	this.certName = CertName(cert)
	// TODO(twifkak): Refactor CertificateMessageFromPEM to be based on the x509.Certificate instead.
	var err error
	this.certMessage, err = certurl.CertificateMessageFromPEM(pemContent)
	if err != nil {
		return nil, errors.Wrap(err, "extracting certificate from CertFile")
	}
	return this, nil
}

func (this CertCache) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	println("path", req.URL.Path)
	if req.URL.Path == path.Join("/", CertURLPrefix, url.PathEscape(this.certName)) {
		// https://tools.ietf.org/html/draft-yasskin-httpbis-origin-signed-exchanges-impl-00#section-3.3
		resp.Header().Set("Content-Type", "application/octet-stream")
		resp.Header().Set("Cache-Control", "public, max-age=604800")
		resp.Header().Set("ETag", "\""+this.certName+"\"")
		// TODO(twifkak): Add cache headers.
		http.ServeContent(resp, req, "", time.Time{}, bytes.NewReader(this.certMessage))
	} else {
		http.NotFound(resp, req)
	}
}
<|endoftext|>