<commit_before>package scanner

import (
	"bufio"
	"bytes"
	"github.com/github/git-media/pointer"
	"github.com/rubyist/tracerx"
	"io"
	"os/exec"
	"strconv"
	"strings"
)

func Scan(ref string) ([]*pointer.Pointer, error) {
	revs, _ := revListStream(ref, ref == "")
	smallShas, _ := catFileBatchCheck(revs)
	pointerc, _ := catFileBatch(smallShas)

	pointers := make([]*pointer.Pointer, 0)
	for p := range pointerc {
		pointers = append(pointers, p)
	}

	return pointers, nil
}

type ScannedPointer struct {
	Name string
	*pointer.Pointer
}

func revListStream(ref string, all bool) (chan string, error) {
	refArgs := []string{"rev-list", "--objects"}
	if all {
		refArgs = append(refArgs, "--all")
	} else {
		refArgs = append(refArgs, ref)
	}

	cmd := exec.Command("git", refArgs...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	tracerx.Printf("run_command: 'git' %s", strings.Join(refArgs, " "))
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	revs := make(chan string)

	go func() {
		scanner := bufio.NewScanner(stdout)
		for scanner.Scan() {
			revs <- scanner.Text()[0:40]
		}
		close(revs)
	}()

	return revs, nil
}

func catFileBatchCheck(revs chan string) (chan string, error) {
	cmd := exec.Command("git", "cat-file", "--batch-check")
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	smallRevs := make(chan string)

	go func() {
		scanner := bufio.NewScanner(stdout)
		for scanner.Scan() {
			line := scanner.Text()
			if line[41:45] == "blob" {
				size, err := strconv.Atoi(line[46:len(line)])
				if err != nil {
					continue
				}
				if size < 200 {
					smallRevs <- line[0:40]
				}
			}
		}
		close(smallRevs)
	}()

	go func() {
		for r := range revs {
			stdin.Write([]byte(r + "\n"))
		}
		stdin.Close()
	}()

	return smallRevs, nil
}

func catFileBatch(revs chan string) (chan *pointer.Pointer, error) {
	cmd := exec.Command("git", "cat-file", "--batch")
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	pointers := make(chan *pointer.Pointer)

	// reads from cat-file stdout, parses out pointers
	go func() {
		bstdout := bufio.NewReader(stdout)
		for {
			l, err := bstdout.ReadBytes('\n')
			if err != nil { // Probably check for EOF
				break
			}

			tracerx.Printf("l: .%s.", string(l))

			fields := bytes.Fields(l)
			s, _ := strconv.Atoi(string(fields[2]))

			nbuf := make([]byte, s)
			_, err = io.ReadFull(bstdout, nbuf)
			if err != nil {
				break // Legit errors
			}

			p, err := pointer.Decode(bytes.NewBuffer(nbuf))
			if err == nil {
				pointers <- p
			}

			_, err = bstdout.ReadBytes('\n') // Extra \n inserted by cat-file
			if err != nil {                  // Probably check for EOF
				break
			}
		}
		close(pointers)
	}()

	// writes shas to cat-file stdin
	go func() {
		for r := range revs {
			stdin.Write([]byte(r + "\n"))
		}
		stdin.Close()
	}()

	return pointers, nil
}
<commit_msg>ンンンンン ンンンン<commit_after>package scanner

import (
	"bufio"
	"bytes"
	"github.com/github/git-media/pointer"
	// "github.com/rubyist/tracerx"
	"io"
	"os/exec"
	"strconv"
)

var blobSizeCutoff = 125

func Scan(ref string) ([]*pointer.Pointer, error) {
	revs, _ := revListStream(ref, ref == "")
	smallShas, _ := catFileBatchCheck(revs)
	pointerc, _ := catFileBatch(smallShas)

	pointers := make([]*pointer.Pointer, 0)
	for p := range pointerc {
		pointers = append(pointers, p)
	}

	return pointers, nil
}

func revListStream(ref string, all bool) (chan string, error) {
	refArgs := []string{"rev-list", "--objects"}
	if all {
		refArgs = append(refArgs, "--all")
	} else {
		refArgs = append(refArgs, ref)
	}

	cmd, err := startCommand("git", refArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	revs := make(chan string)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			revs <- scanner.Text()[0:40]
		}
		close(revs)
	}()

	return revs, nil
}

func catFileBatchCheck(revs chan string) (chan string, error) {
	cmd, err := startCommand("git", "cat-file", "--batch-check")
	if err != nil {
		return nil, err
	}

	smallRevs := make(chan string)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			line := scanner.Text()
			if line[41:45] == "blob" {
				size, err := strconv.Atoi(line[46:len(line)])
				if err != nil {
					continue
				}
				if size < blobSizeCutoff {
					smallRevs <- line[0:40]
				}
			}
		}
		close(smallRevs)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return smallRevs, nil
}

func catFileBatch(revs chan string) (chan *pointer.Pointer, error) {
	cmd, err := startCommand("git", "cat-file", "--batch")
	if err != nil {
		return nil, err
	}

	pointers := make(chan *pointer.Pointer)

	go func() {
		bstdout := bufio.NewReader(cmd.Stdout)
		for {
			l, err := bstdout.ReadBytes('\n')
			if err != nil { // Probably check for EOF
				break
			}

			// Line is formatted:
			// <sha1> <type> <size>
			fields := bytes.Fields(l)
			s, _ := strconv.Atoi(string(fields[2]))

			nbuf := make([]byte, s)
			_, err = io.ReadFull(bstdout, nbuf)
			if err != nil {
				break // Legit errors
			}

			p, err := pointer.Decode(bytes.NewBuffer(nbuf))
			if err == nil {
				pointers <- p
			}

			_, err = bstdout.ReadBytes('\n') // Extra \n inserted by cat-file
			if err != nil {                  // Probably check for EOF
				break
			}
		}
		close(pointers)
	}()

	// writes shas to cat-file stdin
	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return pointers, nil
}

type wrappedCmd struct {
	Stdin  io.WriteCloser
	Stdout io.ReadCloser
	*exec.Cmd
}

func startCommand(command string, args ...string) (*wrappedCmd, error) {
	cmd := exec.Command(command, args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &wrappedCmd{stdin, stdout, cmd}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"reflect"
	"strings"
	"testing"
	"time"
)

func Test_parseCGIHeaders(t *testing.T) {
	data := []struct {
		in      string
		out     string
		headers map[string]string
	}{
		{
			in:      "Some text",
			out:     "Some text",
			headers: map[string]string{},
		},
		{
			in:      "Location: url\n\nSome text",
			out:     "Some text",
			headers: map[string]string{"Location": "url"},
		},
		{
			in:      "Location: url\n\n",
			out:     "",
			headers: map[string]string{"Location": "url"},
		},
		{
			in:      "Location: url\nX-Name:  x-value\n\nSome text",
			out:     "Some text",
			headers: map[string]string{"Location": "url", "X-Name": "x-value"},
		},
		{
			in:      "Some text\nText\n\ntext",
			out:     "Some text\nText\n\ntext",
			headers: map[string]string{},
		},
		{
			in:      "Some text\nText: value in text\n\ntext",
			out:     "Some text\nText: value in text\n\ntext",
			headers: map[string]string{},
		},
		{
			in:      "Text::::\n\ntext",
			out:     "text",
			headers: map[string]string{"Text": ":::"},
		},
		{
			in:      "Text:     :::\n\ntext",
			out:     "text",
			headers: map[string]string{"Text": ":::"},
		},
		{
			in:      "Text:     \n\ntext",
			out:     "Text:     \n\ntext",
			headers: map[string]string{},
		},
		{
			in:      "Header: value\nText:     \n\ntext",
			out:     "Header: value\nText:     \n\ntext",
			headers: map[string]string{},
		},
	}

	for i, item := range data {
		out, headers := parseCGIHeaders(item.in)
		if !reflect.DeepEqual(item.headers, headers) || item.out != out {
			t.Errorf("%d:\nexpected: %s / %#v\nreal    : %s / %#v", i, item.out, item.headers, out, headers)
		}
	}
}

func Test_getShellAndParams(t *testing.T) {
	shell, params, err := getShellAndParams("ls", Config{shell: "sh", defaultShell: "sh", defaultShOpt: "-c"})
	if shell != "sh" || !reflect.DeepEqual(params, []string{"-c", "ls"}) || err != nil {
		t.Errorf("1. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls", Config{shell: "bash", defaultShell: "sh", defaultShOpt: "-c"})
	if shell != "bash" || !reflect.DeepEqual(params, []string{"-c", "ls"}) || err != nil {
		t.Errorf("3. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls -l -a", Config{shell: "", defaultShell: "sh", defaultShOpt: "-c"})
	if shell != "ls" || !reflect.DeepEqual(params, []string{"-l", "-a"}) || err != nil {
		t.Errorf("4. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls -l 'a b'", Config{shell: "", defaultShell: "sh", defaultShOpt: "-c"})
	if shell != "ls" || !reflect.DeepEqual(params, []string{"-l", "a b"}) || err != nil {
		t.Errorf("5. getShellAndParams() failed")
	}

	_, _, err = getShellAndParams("ls '-l", Config{shell: "", defaultShell: "sh", defaultShOpt: "-c"})
	if err == nil {
		t.Errorf("6. getShellAndParams() failed")
	}
}

func Test_getShellAndParams_windows(t *testing.T) {
	shell, params, err := getShellAndParams("ls", Config{shell: "cmd", defaultShell: "cmd", defaultShOpt: "/C"})
	if shell != "cmd" || !reflect.DeepEqual(params, []string{"/C", "ls"}) || err != nil {
		t.Errorf("2. getShellAndParams() failed")
	}
}

func httpRequest(method, url, postData string) ([]byte, error) {
	var postDataReader io.Reader
	if method == "POST" && len(postData) > 0 {
		postDataReader = strings.NewReader(postData)
	}

	request, err := http.NewRequest(method, url, postDataReader)
	if err != nil {
		return nil, err
	}

	request.Header.Set("X-Real-Ip", "127.0.0.1")
	client := &http.Client{}
	res, err := client.Do(request)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return body, nil
}

func getFreePort(t *testing.T) string {
	listen, _ := net.Listen("tcp", ":0")
	parts := strings.Split(listen.Addr().String(), ":")
	err := listen.Close()
	if err != nil {
		t.Errorf("getFreePort() failed")
	}

	return parts[len(parts)-1]
}

func testHTTP(t *testing.T, method, url, postData string, fn func(body string) bool, message string) {
	res, err := httpRequest(method, url, postData)
	if err != nil {
		t.Errorf("%s, get %s failed: %s", message, url, err)
	}
	if !fn(string(res)) {
		t.Errorf("%s failed", message)
	}
}

func Test_main(t *testing.T) {
	port := getFreePort(t)
	os.Args = []string{"shell2http",
		"-add-exit",
		"-cache=1",
		"-cgi",
		// "-export-all-vars",
		"-export-vars=HOME",
		"-one-thread",
		"-shell=",
		"-log=/dev/null",
		"-port=" + port,
		"GET:/echo", "echo 123",
		"POST:/form", "echo var=$v_var",
		"/error", "/ not exists cmd",
		"POST:/post", "cat",
		"/redirect", `echo "Location: /` + "\n" + `"`,
	}
	go main()
	time.Sleep(100 * time.Millisecond) // wait for up http server

	// hide stderr
	oldStderr := os.Stderr // keep backup of the real stderr
	newStderr, err := os.Open("/dev/null")
	if err != nil {
		t.Errorf("open /dev/null: %s", err)
	}
	os.Stderr = newStderr
	defer func() {
		os.Stderr = oldStderr
		err := newStderr.Close()
		if err != nil {
			t.Errorf("Stderr Close failed: %s", err)
		}
	}()

	testHTTP(t, "GET", "http://localhost:"+port+"/", "",
		func(res string) bool { return len(res) > 0 && strings.HasPrefix(res, "<!DOCTYPE html>") },
		"1. get /",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/echo", "",
		func(res string) bool { return res == "123\n" },
		"2. echo",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/echo", "",
		func(res string) bool { return res == "123\n" },
		"3. echo from cache",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/404", "",
		func(res string) bool { return strings.HasPrefix(res, "404 page not found") },
		"4. 404",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/error", "",
		func(res string) bool { return strings.HasPrefix(res, "exec error:") },
		"5. error",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/redirect", "",
		func(res string) bool { return strings.HasPrefix(res, "<!DOCTYPE html>") },
		"6. redirect",
	)

	testHTTP(t, "POST", "http://localhost:"+port+"/post", "X-header: value\n\ntext",
		func(res string) bool { return strings.HasPrefix(res, "text") },
		"7. POST",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/form", "",
		func(res string) bool {
			return strings.HasPrefix(res, http.StatusText(http.StatusMethodNotAllowed))
		},
		"8. POST with GET",
	)
}

func Test_errChain(t *testing.T) {
	err := errChain()
	if err != nil {
		t.Errorf("1. errChain() empty failed")
	}

	err = errChain(func() error { return nil })
	if err != nil {
		t.Errorf("2. errChain() failed")
	}

	err = errChain(func() error { return nil }, func() error { return nil })
	if err != nil {
		t.Errorf("3. errChain() failed")
	}

	err = errChain(func() error { return fmt.Errorf("error") })
	if err == nil {
		t.Errorf("4. errChain() failed")
	}

	err = errChain(func() error { return nil }, func() error { return fmt.Errorf("error") })
	if err == nil {
		t.Errorf("5. errChain() failed")
	}

	var1 := false
	err = errChain(func() error { return fmt.Errorf("error") }, func() error { var1 = true; return nil })
	if err == nil || var1 {
		t.Errorf("6. errChain() failed")
	}
}

func Test_errChainAll(t *testing.T) {
	err := errChainAll()
	if err != nil {
		t.Errorf("1. errChainAll() empty failed")
	}

	err = errChainAll(func() error { return nil })
	if err != nil {
		t.Errorf("2. errChainAll() failed")
	}

	err = errChainAll(func() error { return nil }, func() error { return nil })
	if err != nil {
		t.Errorf("3. errChainAll() failed")
	}

	err = errChainAll(func() error { return fmt.Errorf("error") })
	if err == nil {
		t.Errorf("4. errChainAll() failed")
	}

	err = errChainAll(func() error { return nil }, func() error { return fmt.Errorf("error") })
	if err == nil {
		t.Errorf("5. errChainAll() failed")
	}

	var1 := false
	err = errChainAll(func() error { return fmt.Errorf("error") }, func() error { var1 = true; return nil })
	if err == nil || !var1 {
		t.Errorf("6. errChainAll() failed")
	}
}

func Test_parsePathAndCommands(t *testing.T) {
	tests := []struct {
		name    string
		args    []string
		want    []Command
		wantErr bool
	}{
		{
			name:    "empty list",
			args:    nil,
			want:    nil,
			wantErr: true,
		},
		{
			name:    "empty list 2",
			args:    []string{},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "one arg",
			args:    []string{"arg"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "two arg without path",
			args:    []string{"arg", "arg2"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "three arg",
			args:    []string{"/arg", "date", "aaa"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "two arg",
			args:    []string{"/date", "date"},
			want:    []Command{{path: "/date", cmd: "date"}},
			wantErr: false,
		},
		{
			name:    "four arg",
			args:    []string{"/date", "date", "/", "echo index"},
			want:    []Command{{path: "/date", cmd: "date"}, {path: "/", cmd: "echo index"}},
			wantErr: false,
		},
		{
			name:    "with http method",
			args:    []string{"POST:/date", "date", "GET:/", "echo index"},
			want:    []Command{{path: "/date", cmd: "date", httpMethod: "POST"}, {path: "/", cmd: "echo index", httpMethod: "GET"}},
			wantErr: false,
		},
		{
			name:    "invalid method",
			args:    []string{"get:/date"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "invalid method2",
			args:    []string{"GET_A:/date"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "invalid path",
			args:    []string{"GET:/date 2"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "not uniq path",
			args:    []string{"POST:/date", "date", "POST:/date", "echo index"},
			want:    nil,
			wantErr: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, err := parsePathAndCommands(tt.args)
			if (err != nil) != tt.wantErr {
				t.Errorf("parsePathAndCommands() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("parsePathAndCommands() = %v, want %v", got, tt.want)
			}
		})
	}
}
<commit_msg>Added test case for Windows<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"reflect"
	"strings"
	"testing"
	"time"
)

func Test_parseCGIHeaders(t *testing.T) {
	data := []struct {
		in      string
		out     string
		headers map[string]string
	}{
		{
			in:      "Some text",
			out:     "Some text",
			headers: map[string]string{},
		},
		{
			in:      "Location: url\n\nSome text",
			out:     "Some text",
			headers: map[string]string{"Location": "url"},
		},
		{
			in:      "Location: url\n\n",
			out:     "",
			headers: map[string]string{"Location": "url"},
		},
		{
			in:      "Location: url\nX-Name:  x-value\n\nSome text",
			out:     "Some text",
			headers: map[string]string{"Location": "url", "X-Name": "x-value"},
		},
		{
			in:      "Some text\nText\n\ntext",
			out:     "Some text\nText\n\ntext",
			headers: map[string]string{},
		},
		{
			in:      "Some text\nText: value in text\n\ntext",
			out:     "Some text\nText: value in text\n\ntext",
			headers: map[string]string{},
		},
		{
			in:      "Text::::\n\ntext",
			out:     "text",
			headers: map[string]string{"Text": ":::"},
		},
		{
			in:      "Text:     :::\n\ntext",
			out:     "text",
			headers: map[string]string{"Text": ":::"},
		},
		{
			in:      "Text:     \n\ntext",
			out:     "Text:     \n\ntext",
			headers: map[string]string{},
		},
		{
			in:      "Header: value\nText:     \n\ntext",
			out:     "Header: value\nText:     \n\ntext",
			headers: map[string]string{},
		},
		{
			in:      "Location: url\r\nX-Name:  x-value\r\n\r\nOn Windows",
			out:     "On Windows",
			headers: map[string]string{"Location": "url", "X-Name": "x-value"},
		},
	}

	for i, item := range data {
		out, headers := parseCGIHeaders(item.in)
		if !reflect.DeepEqual(item.headers, headers) || item.out != out {
			t.Errorf("%d:\nexpected: %s / %#v\nreal    : %s / %#v", i, item.out, item.headers, out, headers)
		}
	}
}

func Test_getShellAndParams(t *testing.T) {
	shell, params, err := getShellAndParams("ls", Config{shell: "sh", defaultShell: "sh", defaultShOpt: "-c"})
	if shell != "sh" || !reflect.DeepEqual(params, []string{"-c", "ls"}) || err != nil {
		t.Errorf("1. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls", Config{shell: "bash", defaultShell: "sh", defaultShOpt: "-c"})
	if shell != "bash" || !reflect.DeepEqual(params, []string{"-c", "ls"}) || err != nil {
		t.Errorf("3. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls -l -a", Config{shell: "", defaultShell: "sh", defaultShOpt: "-c"})
	if shell != "ls" || !reflect.DeepEqual(params, []string{"-l", "-a"}) || err != nil {
		t.Errorf("4. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls -l 'a b'", Config{shell: "", defaultShell: "sh", defaultShOpt: "-c"})
	if shell != "ls" || !reflect.DeepEqual(params, []string{"-l", "a b"}) || err != nil {
		t.Errorf("5. getShellAndParams() failed")
	}

	_, _, err = getShellAndParams("ls '-l", Config{shell: "", defaultShell: "sh", defaultShOpt: "-c"})
	if err == nil {
		t.Errorf("6. getShellAndParams() failed")
	}
}

func Test_getShellAndParams_windows(t *testing.T) {
	shell, params, err := getShellAndParams("ls", Config{shell: "cmd", defaultShell: "cmd", defaultShOpt: "/C"})
	if shell != "cmd" || !reflect.DeepEqual(params, []string{"/C", "ls"}) || err != nil {
		t.Errorf("2. getShellAndParams() failed")
	}
}

func httpRequest(method, url, postData string) ([]byte, error) {
	var postDataReader io.Reader
	if method == "POST" && len(postData) > 0 {
		postDataReader = strings.NewReader(postData)
	}

	request, err := http.NewRequest(method, url, postDataReader)
	if err != nil {
		return nil, err
	}

	request.Header.Set("X-Real-Ip", "127.0.0.1")
	client := &http.Client{}
	res, err := client.Do(request)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return body, nil
}

func getFreePort(t *testing.T) string {
	listen, _ := net.Listen("tcp", ":0")
	parts := strings.Split(listen.Addr().String(), ":")
	err := listen.Close()
	if err != nil {
		t.Errorf("getFreePort() failed")
	}

	return parts[len(parts)-1]
}

func testHTTP(t *testing.T, method, url, postData string, fn func(body string) bool, message string) {
	res, err := httpRequest(method, url, postData)
	if err != nil {
		t.Errorf("%s, get %s failed: %s", message, url, err)
	}
	if !fn(string(res)) {
		t.Errorf("%s failed", message)
	}
}

func Test_main(t *testing.T) {
	port := getFreePort(t)
	os.Args = []string{"shell2http",
		"-add-exit",
		"-cache=1",
		"-cgi",
		// "-export-all-vars",
		"-export-vars=HOME",
		"-one-thread",
		"-shell=",
		"-log=/dev/null",
		"-port=" + port,
		"GET:/echo", "echo 123",
		"POST:/form", "echo var=$v_var",
		"/error", "/ not exists cmd",
		"POST:/post", "cat",
		"/redirect", `echo "Location: /` + "\n" + `"`,
	}
	go main()
	time.Sleep(100 * time.Millisecond) // wait for up http server

	// hide stderr
	oldStderr := os.Stderr // keep backup of the real stderr
	newStderr, err := os.Open("/dev/null")
	if err != nil {
		t.Errorf("open /dev/null: %s", err)
	}
	os.Stderr = newStderr
	defer func() {
		os.Stderr = oldStderr
		err := newStderr.Close()
		if err != nil {
			t.Errorf("Stderr Close failed: %s", err)
		}
	}()

	testHTTP(t, "GET", "http://localhost:"+port+"/", "",
		func(res string) bool { return len(res) > 0 && strings.HasPrefix(res, "<!DOCTYPE html>") },
		"1. get /",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/echo", "",
		func(res string) bool { return res == "123\n" },
		"2. echo",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/echo", "",
		func(res string) bool { return res == "123\n" },
		"3. echo from cache",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/404", "",
		func(res string) bool { return strings.HasPrefix(res, "404 page not found") },
		"4. 404",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/error", "",
		func(res string) bool { return strings.HasPrefix(res, "exec error:") },
		"5. error",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/redirect", "",
		func(res string) bool { return strings.HasPrefix(res, "<!DOCTYPE html>") },
		"6. redirect",
	)

	testHTTP(t, "POST", "http://localhost:"+port+"/post", "X-header: value\n\ntext",
		func(res string) bool { return strings.HasPrefix(res, "text") },
		"7. POST",
	)

	testHTTP(t, "GET", "http://localhost:"+port+"/form", "",
		func(res string) bool {
			return strings.HasPrefix(res, http.StatusText(http.StatusMethodNotAllowed))
		},
		"8. POST with GET",
	)
}

func Test_errChain(t *testing.T) {
	err := errChain()
	if err != nil {
		t.Errorf("1. errChain() empty failed")
	}

	err = errChain(func() error { return nil })
	if err != nil {
		t.Errorf("2. errChain() failed")
	}

	err = errChain(func() error { return nil }, func() error { return nil })
	if err != nil {
		t.Errorf("3. errChain() failed")
	}

	err = errChain(func() error { return fmt.Errorf("error") })
	if err == nil {
		t.Errorf("4. errChain() failed")
	}

	err = errChain(func() error { return nil }, func() error { return fmt.Errorf("error") })
	if err == nil {
		t.Errorf("5. errChain() failed")
	}

	var1 := false
	err = errChain(func() error { return fmt.Errorf("error") }, func() error { var1 = true; return nil })
	if err == nil || var1 {
		t.Errorf("6. errChain() failed")
	}
}

func Test_errChainAll(t *testing.T) {
	err := errChainAll()
	if err != nil {
		t.Errorf("1. errChainAll() empty failed")
	}

	err = errChainAll(func() error { return nil })
	if err != nil {
		t.Errorf("2. errChainAll() failed")
	}

	err = errChainAll(func() error { return nil }, func() error { return nil })
	if err != nil {
		t.Errorf("3. errChainAll() failed")
	}

	err = errChainAll(func() error { return fmt.Errorf("error") })
	if err == nil {
		t.Errorf("4. errChainAll() failed")
	}

	err = errChainAll(func() error { return nil }, func() error { return fmt.Errorf("error") })
	if err == nil {
		t.Errorf("5. errChainAll() failed")
	}

	var1 := false
	err = errChainAll(func() error { return fmt.Errorf("error") }, func() error { var1 = true; return nil })
	if err == nil || !var1 {
		t.Errorf("6. errChainAll() failed")
	}
}

func Test_parsePathAndCommands(t *testing.T) {
	tests := []struct {
		name    string
		args    []string
		want    []Command
		wantErr bool
	}{
		{
			name:    "empty list",
			args:    nil,
			want:    nil,
			wantErr: true,
		},
		{
			name:    "empty list 2",
			args:    []string{},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "one arg",
			args:    []string{"arg"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "two arg without path",
			args:    []string{"arg", "arg2"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "three arg",
			args:    []string{"/arg", "date", "aaa"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "two arg",
			args:    []string{"/date", "date"},
			want:    []Command{{path: "/date", cmd: "date"}},
			wantErr: false,
		},
		{
			name:    "four arg",
			args:    []string{"/date", "date", "/", "echo index"},
			want:    []Command{{path: "/date", cmd: "date"}, {path: "/", cmd: "echo index"}},
			wantErr: false,
		},
		{
			name:    "with http method",
			args:    []string{"POST:/date", "date", "GET:/", "echo index"},
			want:    []Command{{path: "/date", cmd: "date", httpMethod: "POST"}, {path: "/", cmd: "echo index", httpMethod: "GET"}},
			wantErr: false,
		},
		{
			name:    "invalid method",
			args:    []string{"get:/date"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "invalid method2",
			args:    []string{"GET_A:/date"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "invalid path",
			args:    []string{"GET:/date 2"},
			want:    nil,
			wantErr: true,
		},
		{
			name:    "not uniq path",
			args:    []string{"POST:/date", "date", "POST:/date", "echo index"},
			want:    nil,
			wantErr: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, err := parsePathAndCommands(tt.args)
			if (err != nil) != tt.wantErr {
				t.Errorf("parsePathAndCommands() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("parsePathAndCommands() = %v, want %v", got, tt.want)
			}
		})
	}
}
<|endoftext|>


<commit_before>package xpi // import "go.mozilla.org/autograph/signer/xpi"

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"fmt"
	"strings"

	"github.com/pkg/errors"
	"go.mozilla.org/autograph/signer"
	"go.mozilla.org/cose"
)

const (
	// algHeaderValue compresses to 1 for the key "alg"
	algHeaderValue = 1
	// kidHeaderValue compresses to 4 for the key "kid"
	kidHeaderValue = 4
)

// stringToCOSEAlg returns the cose.Algorithm for a string or nil if
// the algorithm isn't implemented
func stringToCOSEAlg(s string) (v *cose.Algorithm) {
	switch strings.ToUpper(s) {
	case cose.PS256.Name:
		v = cose.PS256
	case cose.ES256.Name:
		v = cose.ES256
	case cose.ES384.Name:
		v = cose.ES384
	case cose.ES512.Name:
		v = cose.ES512
	default:
		v = nil
	}
	return v
}

// generateIssuerEEKeyPair returns a public and private key pair for
// the provided COSEAlgorithm
func (s *XPISigner) generateCOSEKeyPair(coseAlg *cose.Algorithm) (eeKey crypto.PrivateKey, eePublicKey crypto.PublicKey, err error) {
	var signer *cose.Signer

	switch coseAlg {
	case nil:
		err = fmt.Errorf("Cannot generate private key for nil cose Algorithm")
	case cose.PS256:
		const size = 2048
		eeKey, err = s.getRsaKey(size)
		if err != nil {
			err = errors.Wrapf(err, "failed to generate rsa private key of size %d", size)
			return
		}
		eePublicKey = eeKey.(*rsa.PrivateKey).Public()
	case cose.ES256:
		fallthrough
	case cose.ES384:
		fallthrough
	case cose.ES512:
		signer, err = cose.NewSigner(coseAlg, nil)
		if err != nil {
			err = errors.Wrapf(err, "failed to generate private key")
			return
		}
		eeKey = signer.PrivateKey
		eePublicKey = eeKey.(*ecdsa.PrivateKey).Public()
	}
	return
}

// isSupportedCOSEAlgValue returns whether the COSE alg value is supported or not
func isSupportedCOSEAlgValue(algValue interface{}) bool {
	return algValue == cose.PS256.Value || \
	algValue == cose.ES256.Value || \
	algValue == cose.ES384.Value || \
	algValue == cose.ES512.Value
}

// isValidCOSESignature checks whether a COSE signature is valid for XPIs
func isValidCOSESignature(sig cose.Signature) (eeCert *x509.Certificate, resultErr error) {
	if len(sig.Headers.Unprotected) != 0 {
		resultErr = fmt.Errorf("xpi: COSE Signature must have an empty Unprotected Header")
		return
	}

	if len(sig.Headers.Protected) != 2 {
		resultErr = fmt.Errorf("xpi: COSE Signature must have exactly two Protected Headers")
		return
	}
	algValue, ok := sig.Headers.Protected[algHeaderValue]
	if !ok {
		resultErr = fmt.Errorf("xpi: COSE Signature must have alg in Protected Headers")
		return
	}
	if !isSupportedCOSEAlgValue(algValue) {
		resultErr = fmt.Errorf("xpi: COSE Signature alg %v is not supported", algValue)
		return
	}

	kidValue, ok := sig.Headers.Protected[kidHeaderValue]
	if !ok {
		resultErr = fmt.Errorf("xpi: COSE Signature must have kid in Protected Headers")
		return
	}
	kidBytes, ok := kidValue.([]byte)
	if !ok {
		resultErr = fmt.Errorf("xpi: COSE Signature kid value is not bytes")
		return
	}

	eeCert, err := x509.ParseCertificate(kidBytes) // eeCert
	if err != nil {
		resultErr = errors.Wrapf(err, "xpi: failed to parse X509 EE certificate from COSE Signature")
		return
	}
	return
}

// isValidCOSEMessage checks whether a COSE SignMessage is a valid for
// XPIs and returns parsed intermediate and end entity certs
func isValidCOSEMessage(msg cose.SignMessage) (intermediateCerts, eeCerts []*x509.Certificate, resultErr error) {
	if msg.Payload != nil {
		resultErr = fmt.Errorf("Expected SignMessage payload to be nil, but got %v", msg.Payload)
		return
	}
	if len(msg.Headers.Unprotected) != 0 {
		resultErr = fmt.Errorf("Expected SignMessage Unprotected headers to be empty, but got %v", msg.Headers.Unprotected)
		return
	}

	if len(msg.Headers.Protected) != 1 {
		resultErr = fmt.Errorf("Expected SignMessage Protected headers must contain one value, but got %d", len(msg.Headers.Protected))
		return
	}
	kidValue, ok := msg.Headers.Protected[kidHeaderValue]
	if !ok {
		resultErr = fmt.Errorf("Expected SignMessage must have kid in Protected Headers")
		return
	}
	// check that all kid values are bytes and decode into certs
	kidArray, ok := kidValue.([]interface{})
	if !ok {
		resultErr = fmt.Errorf("Expected SignMessage Protected Headers kid value to be an array got %v with type %T", kidValue, kidValue)
		return
	}
	for i, cert := range kidArray {
		certBytes, ok := cert.([]byte)
		if !ok {
			resultErr = fmt.Errorf("Expected SignMessage Protected Headers kid value %d to be a byte slice got %v with type %T", i, cert, cert)
			return
		}
		intermediateCert, err := x509.ParseCertificate(certBytes)
		if err != nil {
			resultErr = errors.Wrapf(err, "SignMessage Signature Protected Headers kid value %d does not decode to a parseable X509 cert", i)
			return
		}
		intermediateCerts = append(intermediateCerts, intermediateCert)
	}

	for i, sig := range msg.Signatures {
		eeCert, err := isValidCOSESignature(sig)
		if err != nil {
			resultErr = errors.Wrapf(err, "cose signature %d is invalid", i)
			return
		}
		eeCerts = append(eeCerts, eeCert)
	}

	return
}

// verifyCOSESignatures checks that:
//
// 1) COSE manifest and signature files are present
// 2) the PKCS7 manifest is present
// 3) the COSE and PKCS7 manifests do not include COSE files
// 4) we can decode the COSE signature and it has the right format for an XPI
// 5) the right number of signatures are present and all intermediate and end entity certs parse properly
// TODO: 6) there is a trusted path from the included COSE EE certs to the signer cert using the provided intermediates
//
func verifyCOSESignatures(signedFile signer.SignedFile, signOptions Options) error {
	coseManifestBytes, err := readFileFromZIP(signedFile, "META-INF/cose.manifest")
	if err != nil {
		return fmt.Errorf("failed to read META-INF/cose.manifest from signed zip: %v", err)
	}
	coseMsgBytes, err := readFileFromZIP(signedFile, "META-INF/cose.sig")
	if err != nil {
		return fmt.Errorf("failed to read META-INF/cose.sig from signed zip: %v", err)
	}
	pkcs7ManifestBytes, err := readFileFromZIP(signedFile, "META-INF/manifest.mf")
	if err != nil {
		return fmt.Errorf("failed to read META-INF/manifest.mf from signed zip: %v", err)
	}
	coseManifest := string(coseManifestBytes)
	pkcs7Manifest := string(pkcs7ManifestBytes)

	if !strings.Contains(pkcs7Manifest, "cose") {
		return fmt.Errorf("pkcs7 manifest does not contain cose files: %s", pkcs7Manifest)
	}
	if strings.Contains(coseManifest, "cose") {
		return fmt.Errorf("cose manifest contains cose files: %s", coseManifest)
	}

	xpiSig, err := Unmarshal(string(coseMsgBytes), nil)
	if err != nil {
		return errors.Wrap(err, "error unmarshaling cose.sig")
	}
	coseMsg := *xpiSig.signMessage

	if len(coseMsg.Signatures) != len(signOptions.COSEAlgorithms) {
		return fmt.Errorf("cose.sig contains %d signatures, but expected %d", len(coseMsg.Signatures), len(signOptions.COSEAlgorithms))
	}

	_, _, err = isValidCOSEMessage(coseMsg)
	if err != nil {
		return errors.Wrap(err, "cose.sig is not a valid COSE SignMessage")
	}

	return nil
}

// issueCOSESignature returns a CBOR-marshalled COSE SignMessage
// after generating EE certs and signatures for the COSE algorithms
func (s *XPISigner) issueCOSESignature(cn string, manifest []byte, algs []*cose.Algorithm) (coseSig []byte, err error) {
	if s == nil {
		return nil, errors.New("Cannot issue COSE Signature from nil XPISigner")
	}
	if s.issuerCert == nil {
		return nil, errors.New("Cannot issue COSE Signature when XPISigner.issuerCert is nil")
	}

	var (
		coseSigners []cose.Signer
		tmp         = cose.NewSignMessage()
		msg         = &tmp
	)
	msg.Payload = manifest

	// Add list of DER encoded intermediate certificates as message key id
	msg.Headers.Protected["kid"] = [][]byte{s.issuerCert.Raw[:]}

	for _, alg := range algs {
		// create a cert and key
		eeCert, eeKey, err := s.MakeEndEntity(cn, alg)
		if err != nil {
			return nil, err
		}

		// create a COSE.Signer
		signer, err := cose.NewSignerFromKey(alg, eeKey)
		if err != nil {
			return nil, errors.Wrap(err, "xpi: COSE signer creation failed")
		}
		coseSigners = append(coseSigners, *signer)

		// create a COSE Signature holder
		sig := cose.NewSignature()
		sig.Headers.Protected["alg"] = alg.Name
		sig.Headers.Protected["kid"] = eeCert.Raw[:]
		msg.AddSignature(sig)
	}

	// external_aad data must be nil and not byte("")
	err = msg.Sign(rand.Reader, nil, coseSigners)
	if err != nil {
		return nil, errors.Wrap(err, "xpi: COSE signing failed")
	}
	// for addons the signature is detached and the payload is always nil / null
	msg.Payload = nil

	coseSig, err = cose.Marshal(msg)
	if err != nil {
		return nil, errors.Wrap(err, "xpi: error serializing COSE signatures to CBOR")
	}

	return
}
<commit_msg>xpi: fix "invalid character U+005C '\'" errors<commit_after>package xpi // import "go.mozilla.org/autograph/signer/xpi"

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"fmt"
	"strings"

	"github.com/pkg/errors"
	"go.mozilla.org/autograph/signer"
	"go.mozilla.org/cose"
)

const (
	// algHeaderValue compresses to 1 for the key "alg"
	algHeaderValue = 1
	// kidHeaderValue compresses to 4 for the key "kid"
	kidHeaderValue = 4
)

// stringToCOSEAlg returns the cose.Algorithm for a string or nil if
// the algorithm isn't implemented
func stringToCOSEAlg(s string) (v *cose.Algorithm) {
	switch strings.ToUpper(s) {
	case cose.PS256.Name:
		v = cose.PS256
	case cose.ES256.Name:
		v = cose.ES256
	case cose.ES384.Name:
		v = cose.ES384
	case cose.ES512.Name:
		v = cose.ES512
	default:
		v = nil
	}
	return v
}

// generateIssuerEEKeyPair returns a public and private key pair for
// the provided COSEAlgorithm
func (s *XPISigner) generateCOSEKeyPair(coseAlg *cose.Algorithm) (eeKey crypto.PrivateKey, eePublicKey crypto.PublicKey, err error) {
	var signer *cose.Signer

	switch coseAlg {
	case nil:
		err = fmt.Errorf("Cannot generate private key for nil cose Algorithm")
	case cose.PS256:
		const size = 2048
		eeKey, err = s.getRsaKey(size)
		if err != nil {
			err = errors.Wrapf(err, "failed to generate rsa private key of size %d", size)
			return
		}
		eePublicKey = eeKey.(*rsa.PrivateKey).Public()
	case cose.ES256:
		fallthrough
	case cose.ES384:
		fallthrough
	case cose.ES512:
		signer, err = cose.NewSigner(coseAlg, nil)
		if err != nil {
			err = errors.Wrapf(err, "failed to generate private key")
			return
		}
		eeKey = signer.PrivateKey
		eePublicKey = eeKey.(*ecdsa.PrivateKey).Public()
	}
	return
}

// isSupportedCOSEAlgValue returns whether the COSE alg value is supported or not
func isSupportedCOSEAlgValue(algValue interface{}) bool {
	return (algValue == cose.PS256.Value ||
		algValue == cose.ES256.Value ||
		algValue == cose.ES384.Value ||
		algValue == cose.ES512.Value)
}

// isValidCOSESignature checks whether a COSE signature is valid for XPIs
func isValidCOSESignature(sig cose.Signature) (eeCert *x509.Certificate, resultErr error) {
	if len(sig.Headers.Unprotected) != 0 {
		resultErr = fmt.Errorf("xpi: COSE Signature must have an empty Unprotected Header")
		return
	}

	if len(sig.Headers.Protected) != 2 {
		resultErr = fmt.Errorf("xpi: COSE Signature must have exactly two Protected Headers")
		return
	}
	algValue, ok := sig.Headers.Protected[algHeaderValue]
	if !ok {
		resultErr = fmt.Errorf("xpi: COSE Signature must have alg in Protected Headers")
		return
	}
	if !isSupportedCOSEAlgValue(algValue) {
		resultErr = fmt.Errorf("xpi: COSE Signature alg %v is not supported", algValue)
		return
	}

	kidValue, ok := sig.Headers.Protected[kidHeaderValue]
	if !ok {
		resultErr = fmt.Errorf("xpi: COSE Signature must have kid in Protected Headers")
		return
	}
	kidBytes, ok := kidValue.([]byte)
	if !ok {
		resultErr = fmt.Errorf("xpi: COSE Signature kid value is not bytes")
		return
	}

	eeCert, err := x509.ParseCertificate(kidBytes) // eeCert
	if err != nil {
		resultErr = errors.Wrapf(err, "xpi: failed to parse X509 EE certificate from COSE Signature")
		return
	}
	return
}

// isValidCOSEMessage checks whether a COSE SignMessage is a valid for
// XPIs and returns parsed intermediate and end entity certs
func isValidCOSEMessage(msg cose.SignMessage) (intermediateCerts, eeCerts []*x509.Certificate, resultErr error) {
	if msg.Payload != nil {
		resultErr = fmt.Errorf("Expected SignMessage payload to be nil, but got %v", msg.Payload)
		return
	}
	if len(msg.Headers.Unprotected) != 0 {
		resultErr = fmt.Errorf("Expected SignMessage Unprotected headers to be empty, but got %v", msg.Headers.Unprotected)
		return
	}

	if len(msg.Headers.Protected) != 1 {
		resultErr = fmt.Errorf("Expected SignMessage Protected headers must contain one value, but got %d", len(msg.Headers.Protected))
		return
	}
	kidValue, ok := msg.Headers.Protected[kidHeaderValue]
	if !ok {
		resultErr = fmt.Errorf("Expected SignMessage must have kid in Protected Headers")
		return
	}
	// check that all kid values are bytes and decode into certs
	kidArray, ok := kidValue.([]interface{})
	if !ok {
		resultErr = fmt.Errorf("Expected SignMessage Protected Headers kid value to be an array got %v with type %T", kidValue, kidValue)
		return
	}
	for i, cert := range kidArray {
		certBytes, ok := cert.([]byte)
		if !ok {
			resultErr = fmt.Errorf("Expected SignMessage Protected Headers kid value %d to be a byte slice got %v with type %T", i, cert, cert)
			return
		}
		intermediateCert, err := x509.ParseCertificate(certBytes)
		if err != nil {
			resultErr = errors.Wrapf(err, "SignMessage Signature Protected Headers kid value %d does not decode to a parseable X509 cert", i)
			return
		}
		intermediateCerts = append(intermediateCerts, intermediateCert)
	}

	for i, sig := range msg.Signatures {
		eeCert, err := isValidCOSESignature(sig)
		if err != nil {
			resultErr = errors.Wrapf(err, "cose signature %d is invalid", i)
			return
		}
		eeCerts = append(eeCerts, eeCert)
	}

	return
}

// verifyCOSESignatures checks that:
//
// 1) COSE manifest and signature files are present
// 2) the PKCS7 manifest is present
// 3) the COSE and PKCS7 manifests do not include COSE files
// 4) we can decode the COSE signature and it has the right format for an XPI
// 5) the right number of signatures are present and all intermediate and end entity certs parse properly
// TODO: 6) there is a trusted path from the included COSE EE certs to the signer cert using the provided intermediates
//
func verifyCOSESignatures(signedFile signer.SignedFile, signOptions Options) error {
	coseManifestBytes, err := readFileFromZIP(signedFile, "META-INF/cose.manifest")
	if err != nil {
		return fmt.Errorf("failed to read META-INF/cose.manifest from signed zip: %v", err)
	}
	coseMsgBytes, err := readFileFromZIP(signedFile, "META-INF/cose.sig")
	if err != nil {
		return fmt.Errorf("failed to read META-INF/cose.sig from signed zip: %v", err)
	}
	pkcs7ManifestBytes, err := readFileFromZIP(signedFile, "META-INF/manifest.mf")
	if err != nil {
		return fmt.Errorf("failed to read META-INF/manifest.mf from signed zip: %v", err)
	}
	coseManifest := string(coseManifestBytes)
	pkcs7Manifest := string(pkcs7ManifestBytes)

	if !strings.Contains(pkcs7Manifest, "cose") {
		return fmt.Errorf("pkcs7 manifest does not contain cose files: %s", pkcs7Manifest)
	}
	if strings.Contains(coseManifest, "cose") {
		return fmt.Errorf("cose manifest contains cose files: %s", coseManifest)
	}

	xpiSig, err := Unmarshal(string(coseMsgBytes), nil)
	if err != nil {
		return errors.Wrap(err, "error unmarshaling cose.sig")
	}
	coseMsg := *xpiSig.signMessage

	if len(coseMsg.Signatures) != len(signOptions.COSEAlgorithms) {
		return fmt.Errorf("cose.sig contains %d signatures, but expected %d", len(coseMsg.Signatures), len(signOptions.COSEAlgorithms))
	}

	_, _, err = isValidCOSEMessage(coseMsg)
	if err != nil {
		return errors.Wrap(err, "cose.sig is not a valid COSE SignMessage")
	}

	return nil
}

// issueCOSESignature returns a CBOR-marshalled COSE SignMessage
// after generating EE certs and signatures for the COSE algorithms
func (s *XPISigner) issueCOSESignature(cn string, manifest []byte, algs []*cose.Algorithm) (coseSig []byte, err error) {
	if s == nil {
		return nil, errors.New("Cannot issue COSE Signature from nil XPISigner")
	}
	if s.issuerCert == nil {
		return nil, errors.New("Cannot issue COSE Signature when XPISigner.issuerCert is nil")
	}

	var (
		coseSigners []cose.Signer
		tmp         = cose.NewSignMessage()
		msg         = &tmp
	)
	msg.Payload = manifest

	// Add list of DER encoded intermediate certificates as message key id
	msg.Headers.Protected["kid"] = [][]byte{s.issuerCert.Raw[:]}

	for _, alg := range algs {
		// create a cert and key
		eeCert, eeKey, err := s.MakeEndEntity(cn, alg)
		if err != nil {
			return nil, err
		}

		// create a COSE.Signer
		signer, err := cose.NewSignerFromKey(alg, eeKey)
		if err != nil {
			return nil, errors.Wrap(err, "xpi: COSE signer creation failed")
		}
		coseSigners = append(coseSigners, *signer)

		// create a COSE Signature holder
		sig := cose.NewSignature()
		sig.Headers.Protected["alg"] = alg.Name
		sig.Headers.Protected["kid"] = eeCert.Raw[:]
		msg.AddSignature(sig)
	}

	// external_aad data must be nil and not byte("")
	err = msg.Sign(rand.Reader, nil, coseSigners)
	if err != nil {
		return nil, errors.Wrap(err, "xpi: COSE signing failed")
	}
	// for addons the signature is detached and the payload is always nil / null
	msg.Payload = nil

	coseSig, err = cose.Marshal(msg)
	if err != nil {
		return nil, errors.Wrap(err, "xpi: error serializing COSE signatures to CBOR")
	}

	return
}
<|endoftext|>


<commit_before>package peer

import (
	"image"
	"image/color"
	"image/draw"
	"log"
	"sync"
	"time"

	"github.com/golang/freetype/truetype"
	"github.com/pankona/gomo-simra/simra/config"
	"golang.org/x/image/font"
	"golang.org/x/image/font/gofont/goregular"
	"golang.org/x/image/math/fixed"
	"golang.org/x/mobile/app"
	"golang.org/x/mobile/asset"
	"golang.org/x/mobile/exp/app/debug"
	"golang.org/x/mobile/exp/f32"
	"golang.org/x/mobile/exp/gl/glutil"
	"golang.org/x/mobile/exp/sprite"
	"golang.org/x/mobile/exp/sprite/clock"
	"golang.org/x/mobile/exp/sprite/glsprite"
	"golang.org/x/mobile/gl"
)

var (
	glPeer    *GLPeer
	startTime = time.Now()
)

// GLPeer represents gl context.
// Singleton.
type GLPeer struct {
	glctx  gl.Context
	images *glutil.Images
	fps    *debug.FPS
	eng    sprite.Engine
	scene  *sprite.Node
	mu     sync.Mutex
}

// GetGLPeer returns a instance of GLPeer.
// Since GLPeer is singleton, it is necessary to
// call this function to get GLPeer instance.
func GetGLPeer() *GLPeer {
	LogDebug("IN")
	if glPeer == nil {
		glPeer = &GLPeer{}
	}
	LogDebug("OUT")
	return glPeer
}

// Initialize initializes GLPeer.
// This function must be called inadvance of using GLPeer
func (glpeer *GLPeer) Initialize(glctx gl.Context) {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()
	glpeer.glctx = glctx

	// transparency of png
	glpeer.glctx.Enable(gl.BLEND)
	glpeer.glctx.BlendEquation(gl.FUNC_ADD)
	glpeer.glctx.BlendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA)
	glpeer.images = glutil.NewImages(glctx)
	glpeer.fps = debug.NewFPS(glpeer.images)
	glpeer.initEng()

	LogDebug("OUT")
}

func (glpeer *GLPeer) initEng() {
	if glpeer.eng != nil {
		glpeer.eng.Release()
	}
	glpeer.eng = glsprite.Engine(glpeer.images)
	glpeer.scene = &sprite.Node{}
	glpeer.eng.Register(glpeer.scene)
	glpeer.eng.SetTransform(glpeer.scene, f32.Affine{
		{1, 0, 0},
		{0, 1, 0},
	})
}

type arrangerFunc func(e sprite.Engine, n *sprite.Node, t clock.Time)

func (a arrangerFunc) Arrange(e sprite.Engine, n *sprite.Node, t clock.Time) { a(e, n, t) }

func (glpeer *GLPeer) newNode(fn arrangerFunc) *sprite.Node {
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()
	n := &sprite.Node{Arranger: arrangerFunc(fn)}
	glpeer.eng.Register(n)
	glpeer.scene.AppendChild(n)
	return n
}

func (glpeer *GLPeer) appendChild(n *sprite.Node) {
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()
	glpeer.scene.AppendChild(n)
}

func (glpeer *GLPeer) removeChild(n *sprite.Node) {
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()
	glpeer.scene.RemoveChild(n)
}

// LoadTexture return texture that is loaded by the information of arguments.
// Loaded texture can assign using AddSprite function.
func (glpeer *GLPeer) LoadTexture(assetName string, rect image.Rectangle) sprite.SubTex {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()

	a, err := asset.Open(assetName)
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		err := a.Close()
		if err != nil {
			log.Println(err)
		}
	}()

	img, _, err := image.Decode(a)
	if err != nil {
		log.Fatal(err)
	}
	t, err := glpeer.eng.LoadTexture(img)
	if err != nil {
		log.Fatal(err)
	}

	LogDebug("OUT")
	return sprite.SubTex{T: t, R: rect}
}

// MakeTextureByText createst and return texture by speicied text
// Loaded texture can assign using AddSprite function.
// TODO: font parameterize
func (glpeer *GLPeer) MakeTextureByText(text string, fontsize float64, fontcolor color.RGBA, rect image.Rectangle) sprite.SubTex {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()

	dpi := float64(72)
	width := rect.Dx()
	height := rect.Dy()
	img := image.NewRGBA(image.Rect(0, 0, width, height))

	fg, bg := image.NewUniform(fontcolor), image.Transparent
	draw.Draw(img, img.Bounds(), bg, image.Point{}, draw.Src)

	// Draw the text.
	h := font.HintingNone

	gofont, _ := truetype.Parse(goregular.TTF)

	d := &font.Drawer{
		Dst: img,
		Src: fg,
		Face: truetype.NewFace(gofont, &truetype.Options{
			Size:    fontsize,
			DPI:     dpi,
			Hinting: h,
		}),
	}

	textWidth := d.MeasureString(text)

	d.Dot = fixed.Point26_6{
		X: fixed.I(width/2) - textWidth/2,
		Y: fixed.I(int(fontsize * dpi / 72)),
	}
	d.DrawString(text)

	t, err := glpeer.eng.LoadTexture(img)
	if err != nil {
		log.Fatal(err)
	}

	LogDebug("OUT")
	return sprite.SubTex{T: t, R: rect}
}

// Finalize finalizes GLPeer.
// This is called at termination of application.
func (glpeer *GLPeer) Finalize() {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()

	GetSpriteContainer().RemoveSprites()
	glpeer.eng.Release()
	glpeer.fps.Release()
	glpeer.images.Release()
	glpeer.glctx = nil
	LogDebug("OUT")
}

// Update updates screen.
// This is called 60 times per 1 sec.
func (glpeer *GLPeer) Update(publishFunc func() app.PublishResult) {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()

	if glpeer.glctx == nil {
		return
	}
	glpeer.glctx.ClearColor(0, 0, 0, 1) // black background
	glpeer.glctx.Clear(gl.COLOR_BUFFER_BIT)
	now := clock.Time(time.Since(startTime) * 60 / time.Second)

	glpeer.apply()

	glpeer.eng.Render(glpeer.scene, now, sz)
	if config.DEBUG {
		glpeer.fps.Draw(sz)
	}

	// app.Publish() calls glctx.Flush, it should be called within this mutex locking.
	publishFunc()

	LogDebug("OUT")
}

// Reset resets current gl context.
// All sprites are also cleaned.
// This is called at changing of scene, and
// this function is for clean previous scene.
func (glpeer *GLPeer) Reset() {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()
	GetSpriteContainer().RemoveSprites()
	glpeer.initEng()
	LogDebug("OUT")
}

func (glpeer *GLPeer) apply() {

	snpairs := GetSpriteContainer().spriteNodePairs

	for i := range snpairs {
		sc := snpairs[i]
		if sc.sprite == nil || !sc.inuse {
			continue
		}

		affine := &f32.Affine{
			{1, 0, 0},
			{0, 1, 0},
		}
		affine.Translate(affine,
			sc.sprite.X*desiredScreenSize.scale-sc.sprite.W/2*desiredScreenSize.scale+desiredScreenSize.marginWidth/2,
			(desiredScreenSize.height-sc.sprite.Y)*desiredScreenSize.scale-sc.sprite.H/2*desiredScreenSize.scale+desiredScreenSize.marginHeight/2)
		if sc.sprite.R != 0 {
			affine.Translate(affine,
				0.5*sc.sprite.W*desiredScreenSize.scale,
				0.5*sc.sprite.H*desiredScreenSize.scale)
			affine.Rotate(affine, sc.sprite.R)
			affine.Translate(affine,
				-0.5*sc.sprite.W*desiredScreenSize.scale,
				-0.5*sc.sprite.H*desiredScreenSize.scale)
		}
		affine.Scale(affine,
			sc.sprite.W*desiredScreenSize.scale,
			sc.sprite.H*desiredScreenSize.scale)
		glpeer.eng.SetTransform(sc.node, *affine)
	}
}

// Texture represents a texture object that contains subTex
type Texture struct {
	glPeer *GLPeer
	subTex sprite.SubTex
}

// NewTexture returns a new Texture instance
func (glpeer *GLPeer) NewTexture(s sprite.SubTex) *Texture {
	return &Texture{
		glPeer: GetGLPeer(),
		subTex: s,
	}
}

// ReleaseTexture releases specified texture
func (glpeer *GLPeer) ReleaseTexture(t *Texture) {
	glpeer.mu.Lock()
	glpeer.mu.Unlock()
	t.subTex.T.Release()
}
<commit_msg>add GLer interface for making GLPeer mockable<commit_after>package peer

import (
	"image"
	"image/color"
	"image/draw"
	"log"
	"sync"
	"time"

	"github.com/golang/freetype/truetype"
	"github.com/pankona/gomo-simra/simra/config"
	"golang.org/x/image/font"
	"golang.org/x/image/font/gofont/goregular"
	"golang.org/x/image/math/fixed"
	"golang.org/x/mobile/app"
	"golang.org/x/mobile/asset"
	"golang.org/x/mobile/exp/app/debug"
	"golang.org/x/mobile/exp/f32"
	"golang.org/x/mobile/exp/gl/glutil"
	"golang.org/x/mobile/exp/sprite"
	"golang.org/x/mobile/exp/sprite/clock"
	"golang.org/x/mobile/exp/sprite/glsprite"
	"golang.org/x/mobile/gl"
)

type GLer interface {
	// Initialize initializes GLPeer.
	// This function must be called inadvance of using GLPeer
	Initialize(glctx gl.Context)
	// LoadTexture return texture that is loaded by the information of arguments.
	// Loaded texture can assign using AddSprite function.
	LoadTexture(assetName string, rect image.Rectangle) sprite.SubTex
	// MakeTextureByText createst and return texture by speicied text
	// Loaded texture can assign using AddSprite function.
	// TODO: font parameterize
	MakeTextureByText(text string, fontsize float64, fontcolor color.RGBA, rect image.Rectangle) sprite.SubTex
	// Finalize finalizes GLPeer.
	// This is called at termination of application.
	Finalize()
	// Update updates screen.
	// This is called 60 times per 1 sec.
	Update(publishFunc func() app.PublishResult)
	// Reset resets current gl context.
	// All sprites are also cleaned.
	// This is called at changing of scene, and
	// this function is for clean previous scene.
	Reset()
	// NewTexture returns a new Texture instance
	NewTexture(s sprite.SubTex) *Texture
	// ReleaseTexture releases specified texture
	ReleaseTexture(t *Texture)
}

var (
	glPeer    *GLPeer
	startTime = time.Now()
)

// GLPeer represents gl context.
// Singleton.
type GLPeer struct {
	glctx  gl.Context
	images *glutil.Images
	fps    *debug.FPS
	eng    sprite.Engine
	scene  *sprite.Node
	mu     sync.Mutex
}

// GetGLPeer returns a instance of GLPeer.
// Since GLPeer is singleton, it is necessary to
// call this function to get GLPeer instance.
func GetGLPeer() GLer {
	LogDebug("IN")
	if glPeer == nil {
		glPeer = &GLPeer{}
	}
	LogDebug("OUT")
	return glPeer
}

// Initialize initializes GLPeer.
// This function must be called inadvance of using GLPeer
func (glpeer *GLPeer) Initialize(glctx gl.Context) {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()
	glpeer.glctx = glctx

	// transparency of png
	glpeer.glctx.Enable(gl.BLEND)
	glpeer.glctx.BlendEquation(gl.FUNC_ADD)
	glpeer.glctx.BlendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA)
	glpeer.images = glutil.NewImages(glctx)
	glpeer.fps = debug.NewFPS(glpeer.images)
	glpeer.initEng()

	LogDebug("OUT")
}

func (glpeer *GLPeer) initEng() {
	if glpeer.eng != nil {
		glpeer.eng.Release()
	}
	glpeer.eng = glsprite.Engine(glpeer.images)
	glpeer.scene = &sprite.Node{}
	glpeer.eng.Register(glpeer.scene)
	glpeer.eng.SetTransform(glpeer.scene, f32.Affine{
		{1, 0, 0},
		{0, 1, 0},
	})
}

type arrangerFunc func(e sprite.Engine, n *sprite.Node, t clock.Time)

func (a arrangerFunc) Arrange(e sprite.Engine, n *sprite.Node, t clock.Time) { a(e, n, t) }

func (glpeer *GLPeer) newNode(fn arrangerFunc) *sprite.Node {
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()
	n := &sprite.Node{Arranger: arrangerFunc(fn)}
	glpeer.eng.Register(n)
	glpeer.scene.AppendChild(n)
	return n
}

func (glpeer *GLPeer) appendChild(n *sprite.Node) {
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()
	glpeer.scene.AppendChild(n)
}

func (glpeer *GLPeer) removeChild(n *sprite.Node) {
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()
	glpeer.scene.RemoveChild(n)
}

// LoadTexture return texture that is loaded by the information of arguments.
// Loaded texture can assign using AddSprite function.
func (glpeer *GLPeer) LoadTexture(assetName string, rect image.Rectangle) sprite.SubTex {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()

	a, err := asset.Open(assetName)
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		err := a.Close()
		if err != nil {
			log.Println(err)
		}
	}()

	img, _, err := image.Decode(a)
	if err != nil {
		log.Fatal(err)
	}
	t, err := glpeer.eng.LoadTexture(img)
	if err != nil {
		log.Fatal(err)
	}

	LogDebug("OUT")
	return sprite.SubTex{T: t, R: rect}
}

// MakeTextureByText createst and return texture by speicied text
// Loaded texture can assign using AddSprite function.
// TODO: font parameterize
func (glpeer *GLPeer) MakeTextureByText(text string, fontsize float64, fontcolor color.RGBA, rect image.Rectangle) sprite.SubTex {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()

	dpi := float64(72)
	width := rect.Dx()
	height := rect.Dy()
	img := image.NewRGBA(image.Rect(0, 0, width, height))

	fg, bg := image.NewUniform(fontcolor), image.Transparent
	draw.Draw(img, img.Bounds(), bg, image.Point{}, draw.Src)

	// Draw the text.
	h := font.HintingNone

	gofont, _ := truetype.Parse(goregular.TTF)

	d := &font.Drawer{
		Dst: img,
		Src: fg,
		Face: truetype.NewFace(gofont, &truetype.Options{
			Size:    fontsize,
			DPI:     dpi,
			Hinting: h,
		}),
	}

	textWidth := d.MeasureString(text)

	d.Dot = fixed.Point26_6{
		X: fixed.I(width/2) - textWidth/2,
		Y: fixed.I(int(fontsize * dpi / 72)),
	}
	d.DrawString(text)

	t, err := glpeer.eng.LoadTexture(img)
	if err != nil {
		log.Fatal(err)
	}

	LogDebug("OUT")
	return sprite.SubTex{T: t, R: rect}
}

// Finalize finalizes GLPeer.
// This is called at termination of application.
func (glpeer *GLPeer) Finalize() {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()

	GetSpriteContainer().RemoveSprites()
	glpeer.eng.Release()
	glpeer.fps.Release()
	glpeer.images.Release()
	glpeer.glctx = nil
	LogDebug("OUT")
}

// Update updates screen.
// This is called 60 times per 1 sec.
func (glpeer *GLPeer) Update(publishFunc func() app.PublishResult) {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()

	if glpeer.glctx == nil {
		return
	}
	glpeer.glctx.ClearColor(0, 0, 0, 1) // black background
	glpeer.glctx.Clear(gl.COLOR_BUFFER_BIT)
	now := clock.Time(time.Since(startTime) * 60 / time.Second)

	glpeer.apply()

	glpeer.eng.Render(glpeer.scene, now, sz)
	if config.DEBUG {
		glpeer.fps.Draw(sz)
	}

	// app.Publish() calls glctx.Flush, it should be called within this mutex locking.
	publishFunc()

	LogDebug("OUT")
}

// Reset resets current gl context.
// All sprites are also cleaned.
// This is called at changing of scene, and
// this function is for clean previous scene.
func (glpeer *GLPeer) Reset() {
	LogDebug("IN")
	glpeer.mu.Lock()
	defer glpeer.mu.Unlock()
	GetSpriteContainer().RemoveSprites()
	glpeer.initEng()
	LogDebug("OUT")
}

func (glpeer *GLPeer) apply() {

	snpairs := GetSpriteContainer().spriteNodePairs

	for i := range snpairs {
		sc := snpairs[i]
		if sc.sprite == nil || !sc.inuse {
			continue
		}

		affine := &f32.Affine{
			{1, 0, 0},
			{0, 1, 0},
		}
		affine.Translate(affine,
			sc.sprite.X*desiredScreenSize.scale-sc.sprite.W/2*desiredScreenSize.scale+desiredScreenSize.marginWidth/2,
			(desiredScreenSize.height-sc.sprite.Y)*desiredScreenSize.scale-sc.sprite.H/2*desiredScreenSize.scale+desiredScreenSize.marginHeight/2)
		if sc.sprite.R != 0 {
			affine.Translate(affine,
				0.5*sc.sprite.W*desiredScreenSize.scale,
				0.5*sc.sprite.H*desiredScreenSize.scale)
			affine.Rotate(affine, sc.sprite.R)
			affine.Translate(affine,
				-0.5*sc.sprite.W*desiredScreenSize.scale,
				-0.5*sc.sprite.H*desiredScreenSize.scale)
		}
		affine.Scale(affine,
			sc.sprite.W*desiredScreenSize.scale,
			sc.sprite.H*desiredScreenSize.scale)
		glpeer.eng.SetTransform(sc.node, *affine)
	}
}

// Texture represents a texture object that contains subTex
type Texture struct {
	glPeer *GLPeer
	subTex sprite.SubTex
}

// NewTexture returns a new Texture instance
func (glpeer *GLPeer) NewTexture(s sprite.SubTex) *Texture {
	return &Texture{
		glPeer: glpeer,
		subTex: s,
	}
}

// ReleaseTexture releases specified texture
func (glpeer *GLPeer) ReleaseTexture(t *Texture) {
	glpeer.mu.Lock()
	glpeer.mu.Unlock()
	t.subTex.T.Release()
}
<|endoftext|>


<commit_before>package schemas

import (
	"encoding/json"
	"fmt"
	"net/url"
	"reflect"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
	"github.com/ninjasphere/gojsonschema"
	"github.com/xeipuuv/gojsonreference"
)

var log = logger.GetLogger("schemas")

var root = "http://schema.ninjablocks.com/"
var rootURL, _ = url.Parse(root)
var filePrefix = config.MustString("installDirectory") + "/sphere-schemas/"
var fileSuffix = ".json"

var schemaPool = gojsonschema.NewSchemaPool()
var validationEnabled = config.Bool(false, "validate")

func init() {
	schemaPool.FilePrefix = &filePrefix
	schemaPool.FileSuffix = &fileSuffix

	if validationEnabled {
		log.Infof("-------- VALIDATION ENABLED --------")
	}
}

func Validate(schema string, obj interface{}) (*string, error) {

	if !validationEnabled {
		return nil, nil
	}

	jsonBytes, _ := json.Marshal(obj)
	var jsonPayload interface{}
	_ = json.Unmarshal(jsonBytes, &jsonPayload)

	log.Debugf("schema-validator: validating %s %s", schema, jsonBytes)

	doc, err := GetSchema(schema)

	if err != nil {
		return nil, fmt.Errorf("Failed to get document: %s", err)
	}

	// Try to validate the Json against the schema
	result := doc.Validate(jsonPayload)

	// Deal with result
	if !result.Valid() {
		messages := ""

		// Loop through errors
		for _, desc := range result.Errors() {
			messages += fmt.Sprintf("%s\n", desc)
		}
		return &messages, nil
	} else {
		return nil, nil
	}

}

func GetServiceMethods(service string) ([]string, error) {
	doc, err := GetDocument(service+"#/methods", true)

	if err != nil && fmt.Sprintf("%s", err) != "Object has no key 'methods'" {
		return nil, fmt.Errorf("Failed to load schema %s : %s", service, err)
	}

	methods := make([]string, 0, len(doc))
	for method := range doc {
		methods = append(methods, method)
	}

	return methods, nil
}

type flatItem struct {
	path  []string
	value interface{}
}

func flatten(input interface{}, lpath []string, flattened []flatItem) []flatItem {
	if lpath == nil {
		lpath = []string{}
	}
	if flattened == nil {
		flattened = []flatItem{}
	}

	if reflect.ValueOf(input).Kind() == reflect.Map {
		for rkey, value := range input.(map[string]interface{}) {
			flattened = flatten(value, append(lpath, rkey), flattened)
		}
	} else {
		flattened = append(flattened, flatItem{lpath, input})
	}

	return flattened
}

var timeSeriesPaths = make(map[string]string)

/*
* GetEventTimeSeriesData converts an event payload to 0..n time series data points.
* NOTE: The payload must already have been validated. No validation is done here.
* NOTE: This accepts the json payload. So either a simple type or map[string]interface{}
*
* @param value {interface{}} The payload of the event. Can be null if there is no payload
* @param eventSchemaUri {string} The URI of the schema defining the event (usually ends with #/events/{name})
* @returns {Array} An array of records that need to be saved to a time series db
 */
func GetEventTimeSeriesData(value interface{}, serviceSchemaUri, event string) ([]model.TimeSeriesDatapoint, error) {

	// We don't want a pointer, just grab the actual value
	if reflect.ValueOf(value).Kind() == reflect.Ptr {
		value = reflect.ValueOf(value).Elem().Interface()
	}

	var timeseriesData = make([]model.TimeSeriesDatapoint, 0)

	//log.Debugf("Finding time series data for service: %s event: %s from payload: %v", serviceSchemaUri, event, value)

	flat := flatten(value, nil, nil)

	for _, point := range flat {
		//log.Debugf("-- Checking: %v", point)

		refPath := "#/events/" + event + "/value"

		key := refPath
		if len(point.path) > 0 {
			key = strings.Join(append([]string{refPath}, point.path...), "/properties/")
		}

		//log.Debugf("Created path %s", key)

		var timeseriesType string
		timeseriesType, ok := timeSeriesPaths[key]

		if !ok {

			pointSchema, err := GetDocument(serviceSchemaUri+refPath, true)

			for _, property := range point.path {

				props, ok := pointSchema["properties"].(map[string]interface{})
				if !ok {
					log.Warningf("Unknown property %s in service %s event %s. error: %s", property, serviceSchemaUri, event, err)
					ok = false
				}

				pointSchema, ok = props[property].(map[string]interface{})

				pointSchema, err = resolve(serviceSchemaUri+refPath, pointSchema)

				if !ok {
					log.Warningf("Unknown property %s in service %s event %s. error: %s", property, serviceSchemaUri, event, err)
					ok = false
				}

			}

			if err != nil {
				// As the data has been validated, this *shouldn't* happen. BUT we might be allowing unknown properties through.
				log.Warningf("Unknown property %s in service %s event %s. error: %s", refPath, serviceSchemaUri, event, err)
				ok = false
			} else {
				timeseriesType, ok = pointSchema["timeseries"].(string)
			}

			timeSeriesPaths[key] = timeseriesType
		}

		if ok && timeseriesType != "" {

			dp := model.TimeSeriesDatapoint{
				Path: strings.Join(point.path, "."),
				Type: timeseriesType,
			}

			switch timeseriesType {
			case "value" | "boolean":
				dp.Value = point.value
			}

			// The only other type is 'event', which doesn't have or need a value

			timeseriesData = append(timeseriesData, dp)

		}
	}

	return timeseriesData, nil
}

func GetDocument(documentURL string, resolveRefs bool) (map[string]interface{}, error) {
	resolvedURL, err := resolveUrl(rootURL, documentURL)
	if err != nil {
		return nil, err
	}

	localURL := useLocalUrl(resolvedURL)

	doc, err := schemaPool.GetDocument(localURL)
	if err != nil {
		return nil, err
	}

	refURL, _ := url.Parse(documentURL)

	document := doc.Document

	if err == nil && refURL.Fragment != "" {
		// If we have a fragment, grab it.
		document, _, err = resolvedURL.GetPointer().Get(document)
	}

	if err != nil {
		return nil, err
	}

	mapDoc := document.(map[string]interface{})

	if resolveRefs {

		return resolve(documentURL, mapDoc)
		/*if ref, ok := mapDoc["$ref"]; ok && ref != "" {
			log.Debugf("Got $ref: %s", ref)
			var resolvedRef, err = resolveUrl(resolvedURL.GetUrl(), ref.(string))
			log.Debugf("resolved %s to %s", ref.(string), resolvedRef.GetUrl().String())
			if err != nil {
				return nil, err
			}
			return GetDocument(resolvedRef.String(), true)
		}*/
	}

	return mapDoc, nil
}

func resolve(documentURL string, doc map[string]interface{}) (map[string]interface{}, error) {

	if ref, ok := doc["$ref"]; ok && ref != "" {
		log.Debugf("Got $ref: %s", ref)

		resolvedURL, err := resolveUrl(rootURL, documentURL)
		if err != nil {
			return nil, err
		}

		resolvedRef, err := resolveUrl(resolvedURL.GetUrl(), ref.(string))
		log.Debugf("resolved %s to %s", ref.(string), resolvedRef.GetUrl().String())
		if err != nil {
			return nil, err
		}
		return GetDocument(resolvedRef.String(), true)
	}

	return doc, nil
}

type schemaResponse struct {
	schema *gojsonschema.JsonSchemaDocument
	err    error
}

var schemasCache = make(map[string]schemaResponse)

func GetSchema(documentURL string) (*gojsonschema.JsonSchemaDocument, error) {

	resolved, err := resolveUrl(rootURL, documentURL)
	if err != nil {
		return nil, err
	}
	localRef := useLocalUrl(resolved)
	local := localRef.GetUrl().String()

	schema, ok := schemasCache[local]
	if !ok {
		log.Debugf("Cache miss on '%s'", resolved.GetUrl().String())
		s, err := gojsonschema.NewJsonSchemaDocument(local, schemaPool)
		schema = schemaResponse{s, err}
		schemasCache[local] = schema
	}
	return schema.schema, schema.err
}

func useLocalUrl(ref gojsonreference.JsonReference) gojsonreference.JsonReference {
	// Grab ninjablocks schemas locally

	local := strings.Replace(ref.GetUrl().String(), root, "file:///", 1)
	log.Debugf("Fetching document from %s", local)
	localURL, _ := gojsonreference.NewJsonReference(local)
	return localURL
}

func resolveUrl(root *url.URL, documentURL string) (gojsonreference.JsonReference, error) {
	ref, err := gojsonreference.NewJsonReference(documentURL)
	if err != nil {
		return ref, err
	}
	resolvedURL := root.ResolveReference(ref.GetUrl())

	return gojsonreference.NewJsonReference(resolvedURL.String())
}

func main() {
	//spew.Dump(Validate("/protocol/humidity#/events/state/value", "hello"))
	//spew.Dump(Validate("protocol/humidity#/events/state/value", 10))

	// TODO: FAIL! min/max not taken care of!
	//spew.Dump(Validate("/protocol/humidity#/events/state/value", -10))

	//spew.Dump(GetServiceMethods("/protocol/power"))
	/*	doc, _ := GetDocument("/protocol/humidity", true)
		flattened := flatten(doc, []string{}, make([]flatItem, 0))
		spew.Dump(flattened)*/

	spew.Dump(GetEventTimeSeriesData(10, "/protocol/humidity", "state"))

	var payload = &testVal{
		Rumbling: true,
		X:        0.5,
		Y:        -0.1,
		Z: &testValSize{
			Hello:   10,
			Goodbye: 20,
		},
	}

	jsonBytes, _ := json.Marshal(payload)
	var jsonPayload interface{}
	_ = json.Unmarshal(jsonBytes, &jsonPayload)

	points, _ := GetEventTimeSeriesData(&jsonPayload, "/protocol/game-controller/joystick", "state")

	js, _ := json.Marshal(points)

	log.Infof("Points: %s", js)

	//spew.Dump(GetEventTimeSeriesData(nil, "/protocol/humidity", "state"))
}

type testVal struct {
	Rumbling bool         `json:"rumbling"`
	X        float64      `json:"x"`
	Y        float64      `json:"y"`
	Z        *testValSize `json:"z"`
}

type testValSize struct {
	Hello   int `json:"hello"`
	Goodbye int `json:"goodbye"`
}
<commit_msg>Support for adding tags.<commit_after>package schemas

import (
	"encoding/json"
	"fmt"
	"net/url"
	"reflect"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
	"github.com/ninjasphere/gojsonschema"
	"github.com/xeipuuv/gojsonreference"
)

var log = logger.GetLogger("schemas")

var root = "http://schema.ninjablocks.com/"
var rootURL, _ = url.Parse(root)
var filePrefix = config.MustString("installDirectory") + "/sphere-schemas/"
var fileSuffix = ".json"

var schemaPool = gojsonschema.NewSchemaPool()
var validationEnabled = config.Bool(false, "validate")

func init() {
	schemaPool.FilePrefix = &filePrefix
	schemaPool.FileSuffix = &fileSuffix

	if validationEnabled {
		log.Infof("-------- VALIDATION ENABLED --------")
	}
}

func Validate(schema string, obj interface{}) (*string, error) {

	if !validationEnabled {
		return nil, nil
	}

	jsonBytes, _ := json.Marshal(obj)
	var jsonPayload interface{}
	_ = json.Unmarshal(jsonBytes, &jsonPayload)

	log.Debugf("schema-validator: validating %s %s", schema, jsonBytes)

	doc, err := GetSchema(schema)

	if err != nil {
		return nil, fmt.Errorf("Failed to get document: %s", err)
	}

	// Try to validate the Json against the schema
	result := doc.Validate(jsonPayload)

	// Deal with result
	if !result.Valid() {
		messages := ""

		// Loop through errors
		for _, desc := range result.Errors() {
			messages += fmt.Sprintf("%s\n", desc)
		}
		return &messages, nil
	} else {
		return nil, nil
	}

}

func GetServiceMethods(service string) ([]string, error) {
	doc, err := GetDocument(service+"#/methods", true)

	if err != nil && fmt.Sprintf("%s", err) != "Object has no key 'methods'" {
		return nil, fmt.Errorf("Failed to load schema %s : %s", service, err)
	}

	methods := make([]string, 0, len(doc))
	for method := range doc {
		methods = append(methods, method)
	}

	return methods, nil
}

type flatItem struct {
	path  []string
	value interface{}
}

func flatten(input interface{}, lpath []string, flattened []flatItem) []flatItem {
	if lpath == nil {
		lpath = []string{}
	}
	if flattened == nil {
		flattened = []flatItem{}
	}

	if reflect.ValueOf(input).Kind() == reflect.Map {
		for rkey, value := range input.(map[string]interface{}) {
			flattened = flatten(value, append(lpath, rkey), flattened)
		}
	} else {
		flattened = append(flattened, flatItem{lpath, input})
	}

	return flattened
}

var timeSeriesPaths = make(map[string]string)

/*
* GetEventTimeSeriesData converts an event payload to 0..n time series data points.
* NOTE: The payload must already have been validated. No validation is done here.
* NOTE: This accepts the json payload. So either a simple type or map[string]interface{}
*
* @param value {interface{}} The payload of the event. Can be null if there is no payload
* @param eventSchemaUri {string} The URI of the schema defining the event (usually ends with #/events/{name})
* @returns {Array} An array of records that need to be saved to a time series db
 */
func GetEventTimeSeriesData(value interface{}, serviceSchemaUri, event string) ([]model.TimeSeriesDatapoint, error) {

	// We don't want a pointer, just grab the actual value
	if reflect.ValueOf(value).Kind() == reflect.Ptr {
		value = reflect.ValueOf(value).Elem().Interface()
	}

	var timeseriesData = make([]model.TimeSeriesDatapoint, 0)

	//log.Debugf("Finding time series data for service: %s event: %s from payload: %v", serviceSchemaUri, event, value)

	flat := flatten(value, nil, nil)

	for _, point := range flat {
		//log.Debugf("-- Checking: %v", point)

		refPath := "#/events/" + event + "/value"

		key := refPath
		if len(point.path) > 0 {
			key = strings.Join(append([]string{refPath}, point.path...), "/properties/")
		}

		//log.Debugf("Created path %s", key)

		var timeseriesType string
		timeseriesType, ok := timeSeriesPaths[key]

		if !ok {

			pointSchema, err := GetDocument(serviceSchemaUri+refPath, true)

			for _, property := range point.path {

				props, ok := pointSchema["properties"].(map[string]interface{})
				if !ok {
					log.Warningf("Unknown property %s in service %s event %s. error: %s", property, serviceSchemaUri, event, err)
					ok = false
				}

				pointSchema, ok = props[property].(map[string]interface{})

				pointSchema, err = resolve(serviceSchemaUri+refPath, pointSchema)

				if !ok {
					log.Warningf("Unknown property %s in service %s event %s. error: %s", property, serviceSchemaUri, event, err)
					ok = false
				}

			}

			if err != nil {
				// As the data has been validated, this *shouldn't* happen. BUT we might be allowing unknown properties through.
				log.Warningf("Unknown property %s in service %s event %s. error: %s", refPath, serviceSchemaUri, event, err)
				ok = false
			} else {
				timeseriesType, ok = pointSchema["timeseries"].(string)
			}

			timeSeriesPaths[key] = timeseriesType
		}

		if ok && timeseriesType != "" {

			dp := model.TimeSeriesDatapoint{
				Path: strings.Join(point.path, "."),
				Type: timeseriesType,
			}

			switch timeseriesType {
			case "value", "boolean", "tag":
				dp.Value = point.value
			}

			// The only other type is 'event', which doesn't have or need a value

			timeseriesData = append(timeseriesData, dp)

		}
	}

	return timeseriesData, nil
}

func GetDocument(documentURL string, resolveRefs bool) (map[string]interface{}, error) {
	resolvedURL, err := resolveUrl(rootURL, documentURL)
	if err != nil {
		return nil, err
	}

	localURL := useLocalUrl(resolvedURL)

	doc, err := schemaPool.GetDocument(localURL)
	if err != nil {
		return nil, err
	}

	refURL, _ := url.Parse(documentURL)

	document := doc.Document

	if err == nil && refURL.Fragment != "" {
		// If we have a fragment, grab it.
		document, _, err = resolvedURL.GetPointer().Get(document)
	}

	if err != nil {
		return nil, err
	}

	mapDoc := document.(map[string]interface{})

	if resolveRefs {

		return resolve(documentURL, mapDoc)
		/*if ref, ok := mapDoc["$ref"]; ok && ref != "" {
			log.Debugf("Got $ref: %s", ref)
			var resolvedRef, err = resolveUrl(resolvedURL.GetUrl(), ref.(string))
			log.Debugf("resolved %s to %s", ref.(string), resolvedRef.GetUrl().String())
			if err != nil {
				return nil, err
			}
			return GetDocument(resolvedRef.String(), true)
		}*/
	}

	return mapDoc, nil
}

func resolve(documentURL string, doc map[string]interface{}) (map[string]interface{}, error) {

	if ref, ok := doc["$ref"]; ok && ref != "" {
		log.Debugf("Got $ref: %s", ref)

		resolvedURL, err := resolveUrl(rootURL, documentURL)
		if err != nil {
			return nil, err
		}

		resolvedRef, err := resolveUrl(resolvedURL.GetUrl(), ref.(string))
		log.Debugf("resolved %s to %s", ref.(string), resolvedRef.GetUrl().String())
		if err != nil {
			return nil, err
		}
		return GetDocument(resolvedRef.String(), true)
	}

	return doc, nil
}

type schemaResponse struct {
	schema *gojsonschema.JsonSchemaDocument
	err    error
}

var schemasCache = make(map[string]schemaResponse)

func GetSchema(documentURL string) (*gojsonschema.JsonSchemaDocument, error) {

	resolved, err := resolveUrl(rootURL, documentURL)
	if err != nil {
		return nil, err
	}
	localRef := useLocalUrl(resolved)
	local := localRef.GetUrl().String()

	schema, ok := schemasCache[local]
	if !ok {
		log.Debugf("Cache miss on '%s'", resolved.GetUrl().String())
		s, err := gojsonschema.NewJsonSchemaDocument(local, schemaPool)
		schema = schemaResponse{s, err}
		schemasCache[local] = schema
	}
	return schema.schema, schema.err
}

func useLocalUrl(ref gojsonreference.JsonReference) gojsonreference.JsonReference {
	// Grab ninjablocks schemas locally

	local := strings.Replace(ref.GetUrl().String(), root, "file:///", 1)
	log.Debugf("Fetching document from %s", local)
	localURL, _ := gojsonreference.NewJsonReference(local)
	return localURL
}

func resolveUrl(root *url.URL, documentURL string) (gojsonreference.JsonReference, error) {
	ref, err := gojsonreference.NewJsonReference(documentURL)
	if err != nil {
		return ref, err
	}
	resolvedURL := root.ResolveReference(ref.GetUrl())

	return gojsonreference.NewJsonReference(resolvedURL.String())
}

func main() {
	//spew.Dump(Validate("/protocol/humidity#/events/state/value", "hello"))
	//spew.Dump(Validate("protocol/humidity#/events/state/value", 10))

	// TODO: FAIL! min/max not taken care of!
	//spew.Dump(Validate("/protocol/humidity#/events/state/value", -10))

	//spew.Dump(GetServiceMethods("/protocol/power"))
	/*	doc, _ := GetDocument("/protocol/humidity", true)
		flattened := flatten(doc, []string{}, make([]flatItem, 0))
		spew.Dump(flattened)*/

	spew.Dump(GetEventTimeSeriesData(10, "/protocol/humidity", "state"))

	var payload = &testVal{
		Rumbling: true,
		X:        0.5,
		Y:        -0.1,
		Z: &testValSize{
			Hello:   10,
			Goodbye: 20,
		},
	}

	jsonBytes, _ := json.Marshal(payload)
	var jsonPayload interface{}
	_ = json.Unmarshal(jsonBytes, &jsonPayload)

	points, _ := GetEventTimeSeriesData(&jsonPayload, "/protocol/game-controller/joystick", "state")

	js, _ := json.Marshal(points)

	log.Infof("Points: %s", js)

	//spew.Dump(GetEventTimeSeriesData(nil, "/protocol/humidity", "state"))
}

type testVal struct {
	Rumbling bool         `json:"rumbling"`
	X        float64      `json:"x"`
	Y        float64      `json:"y"`
	Z        *testValSize `json:"z"`
}

type testValSize struct {
	Hello   int `json:"hello"`
	Goodbye int `json:"goodbye"`
}
<|endoftext|>


<commit_before>package scraper

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"strings"
	"sync"
)

// ResourceOrigin represents data originally used to create this resource
type ResourceOrigin struct {
	// URL represents the initial URL received by input
	URL string
	// IP represents the initial IP address received by input
	IP string
	// Host represents the original requested hostname for the resource
	Host string
}

// Response represents the data for the HTTP-based request, closely matching
// http.Response
type Response struct {
	// Host represents the resulting host derived by the original returned
	// resource
	Host string

	// Remote represents if the resulting resource is remote to the original domain
	Remote bool

	// Code represents the numeric HTTP based status code
	Code int

	// Body represents a string implementation of the byte array returned by
	// http.Response. Only used for primary requests, ignored for Resource structs.
	Body string

	// Scheme represents the end scheme used to fetch the page. For example, https
	Scheme string

	// ContentLength represents the number of bytes in the body of the response
	ContentLength int64

	// TLS represents the SSL/TLS handshake/session if the resource was loaded over
	// SSL.
	TLS *tls.ConnectionState
}

// Resource represents a single entity of many within a given crawl. These should
// only be of type css, js, jpg, png, etc (static resources).
type Resource struct {
	// request represents what we were provided before the request
	request ResourceOrigin

	// Response represents the end result/data/status/etc.
	Response Response

	// URL represents the resulting static URL derived by the original result page
	URL string

	// Error represents any errors that may have occurred when fetching the resource
	Error error

	// Time represents the time it took to complete the request
	Time *TimerResult
}

// fetchResource fetches a singular resource from a page, returning a *Resource struct.
// As we don't care much about the body of the resource, that can safely be ignored. We
// must still close the body object, however.
func (c *Crawler) fetchResource(rsrc *Resource) {
	var err error

	defer resourcePool.Done()

	// calculate the time it takes to fetch the request
	resp, err := c.Get(rsrc.request.URL)

	if err != nil {
		rsrc.Error = err
		return
	}

	if resp.Body != nil {
		resp.Body.Close()
	}

	rsrc.request.Host, err = getHost(rsrc.request.URL)
	if err != nil {
		rsrc.Error = err
		return
	}

	rsrc.Response.Host = resp.Request.Host
	rsrc.URL = resp.URL
	rsrc.Response.Code = resp.StatusCode
	rsrc.Response.Scheme = resp.Request.URL.Scheme
	rsrc.Response.ContentLength = resp.ContentLength
	rsrc.Response.TLS = resp.TLS
	rsrc.Time = resp.Time

	if rsrc.Response.Host != rsrc.request.Host {
		rsrc.Response.Remote = true
	}

	c.Log.Printf("fetched %s in %dms with status %d", rsrc.URL, rsrc.Time.Milli, rsrc.Response.Code)

	return
}

// Results -- struct returned by Crawl() to represent the entire crawl process
type Results struct {
	// Inherit the Resource struct
	Resource

	// Slice of Resource structs containing the needed resources for the given URL
	Resources []*Resource

	// ResourceTime shows how long it took to fetch all resources
	ResourceTime *TimerResult

	// TotalTime represents the time it took to crawl the site
	TotalTime *TimerResult
}

func (r *Results) String() string {
	if r.Resources != nil && r.ResourceTime != nil && r.TotalTime != nil {
		return fmt.Sprintf("<url(%s) == %d, resources(%d), resourceTime(%dms), totalTime(%dms), err(%s)>", r.URL, r.Response.Code, len(r.Resources), r.ResourceTime.Milli, r.TotalTime.Milli, r.Error)
	}

	return fmt.Sprintf("<url(%s), ip(%s), err(%s)>", r.request.URL, r.request.IP, r.Error)
}

var resourcePool sync.WaitGroup

// FetchURL manages the fetching of the main resource, as well as all child resources,
// providing a Results struct containing the entire crawl data needed
func (c *Crawler) FetchURL(URL string) (res *Results) {
	res = &Results{}
	crawlTimer := NewTimer()

	var err error

	res.request.URL = URL
	res.request.Host, err = getHost(URL)
	if err != nil {
		res.Error = err
		return
	}
	res.request.IP = c.ipmap[res.request.Host]

	// actually fetch the request
	resp, err := c.Get(URL)

	defer func() {
		crawlTimer.End()
		res.TotalTime = crawlTimer.Result
	}()

	if err != nil {
		res.Error = err
		return
	}

	defer resp.Body.Close()

	res.Response.Host = resp.Request.Host
	res.URL = resp.URL
	res.Response.Code = resp.StatusCode
	res.Response.Scheme = resp.Request.URL.Scheme
	res.Response.ContentLength = resp.ContentLength
	res.Response.TLS = resp.TLS
	res.Time = resp.Time

	if res.Response.Host != res.request.Host {
		res.Response.Remote = true
	}

	buf, _ := ioutil.ReadAll(resp.Body)
	b := ioutil.NopCloser(bytes.NewReader(buf))
	defer b.Close()

	bbytes, err := ioutil.ReadAll(bytes.NewBuffer(buf))
	if err == nil && len(bbytes) != 0 {
		res.Response.Body = string(bbytes[:])
	}

	urls := getSrc(b, resp.Request)

	c.Log.Printf("fetched %s in %dms with status %d", res.URL, res.Time.Milli, res.Response.Code)

	resourceTime := NewTimer()

	defer func() {
		resourceTime.End()
		res.ResourceTime = resourceTime.Result
	}()

	for i := range urls {
		resourcePool.Add(1)

		rsrc := &Resource{request: ResourceOrigin{URL: urls[i]}}
		res.Resources = append(res.Resources, rsrc)
		go c.fetchResource(res.Resources[i])
	}

	resourcePool.Wait()

	return
}

// Domain represents a url we need to fetch, including the items needed to
// fetch said url. E.g: host, port, ip, scheme, path, etc.
type Domain struct {
	URL *url.URL
	IP  string
}

// Crawler is the higher level struct which wraps the entire threaded crawl process
type Crawler struct {
	Log     *log.Logger
	Domains []*Domain
	Results []*Results
	ipmap   map[string]string
}

// Crawl represents the higher level functionality of scraper. Crawl should
// concurrently request the needed resources for a list of domains, allowing
// the bypass of DNS lookups where necessary.
func (c *Crawler) Crawl() {
	var results []*Results
	var wg sync.WaitGroup
	timer := NewTimer()

	// strip all common duplicate domain/ip pairs
	stripDups(&c.Domains)

	c.ipmap = make(map[string]string)
	for i := range c.Domains {
		c.ipmap[c.Domains[i].URL.Host] = c.Domains[i].IP
		c.ipmap[strings.TrimPrefix(c.Domains[i].URL.Host, "www.")] = c.Domains[i].IP // no www. directive
		c.ipmap["www."+c.Domains[i].URL.Host] = c.Domains[i].IP                      // www. directive
	}

	// loop through all supplied urls and send them to a worker to be fetched
	for _, domain := range c.Domains {
		wg.Add(1)

		go func(domain *Domain) {
			defer wg.Done()

			result := c.FetchURL(domain.URL.String())
			results = append(results, result)

			if result.Error != nil {
				c.Log.Printf("error scanning %s (error: %s)", domain.URL.String(), result.Error)
			} else {
				c.Log.Printf("finished scanning %s (%dms)", domain.URL.String(), result.TotalTime.Milli)
			}
		}(domain)
	}

	// wait for all workers to complete their tasks
	wg.Wait()
	timer.End()

	c.Log.Printf("finished scanning %d urls in %d seconds", len(results), timer.Result.Seconds)

	// give some extra details
	var resSuccess, resError int
	for i := range results {
		if results[i].Error != nil {
			resError++
			continue
		}

		resSuccess++
	}

	c.Log.Printf("%d successful, %d errored", resSuccess, resError)

	c.Results = results

	return
}

// GetResults gets the potential results of a given requested url/ip
func (c *Crawler) GetResults(URL, IP string) *Results {
	for i := range c.Results {
		if c.Results[i].request.URL == URL && c.Results[i].request.IP == IP {
			return c.Results[i]
		}
	}

	return nil
}
<commit_msg>fix www. redirect cornercase<commit_after>package scraper

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"strings"
	"sync"
)

// ResourceOrigin represents data originally used to create this resource
type ResourceOrigin struct {
	// URL represents the initial URL received by input
	URL string
	// IP represents the initial IP address received by input
	IP string
	// Host represents the original requested hostname for the resource
	Host string
}

// Response represents the data for the HTTP-based request, closely matching
// http.Response
type Response struct {
	// Host represents the resulting host derived by the original returned
	// resource
	Host string

	// Remote represents if the resulting resource is remote to the original domain
	Remote bool

	// Code represents the numeric HTTP based status code
	Code int

	// Body represents a string implementation of the byte array returned by
	// http.Response. Only used for primary requests, ignored for Resource structs.
	Body string

	// Scheme represents the end scheme used to fetch the page. For example, https
	Scheme string

	// ContentLength represents the number of bytes in the body of the response
	ContentLength int64

	// TLS represents the SSL/TLS handshake/session if the resource was loaded over
	// SSL.
	TLS *tls.ConnectionState
}

// Resource represents a single entity of many within a given crawl. These should
// only be of type css, js, jpg, png, etc (static resources).
type Resource struct {
	// request represents what we were provided before the request
	request ResourceOrigin

	// Response represents the end result/data/status/etc.
	Response Response

	// URL represents the resulting static URL derived by the original result page
	URL string

	// Error represents any errors that may have occurred when fetching the resource
	Error error

	// Time represents the time it took to complete the request
	Time *TimerResult
}

// fetchResource fetches a singular resource from a page, returning a *Resource struct.
// As we don't care much about the body of the resource, that can safely be ignored. We
// must still close the body object, however.
func (c *Crawler) fetchResource(rsrc *Resource) {
	var err error

	defer resourcePool.Done()

	// calculate the time it takes to fetch the request
	resp, err := c.Get(rsrc.request.URL)

	if err != nil {
		rsrc.Error = err
		return
	}

	if resp.Body != nil {
		resp.Body.Close()
	}

	rsrc.request.Host, err = getHost(rsrc.request.URL)
	if err != nil {
		rsrc.Error = err
		return
	}

	rsrc.Response.Host = resp.Request.Host
	rsrc.URL = resp.URL
	rsrc.Response.Code = resp.StatusCode
	rsrc.Response.Scheme = resp.Request.URL.Scheme
	rsrc.Response.ContentLength = resp.ContentLength
	rsrc.Response.TLS = resp.TLS
	rsrc.Time = resp.Time

	if rsrc.Response.Host != rsrc.request.Host {
		rsrc.Response.Remote = true
	}

	c.Log.Printf("fetched %s in %dms with status %d", rsrc.URL, rsrc.Time.Milli, rsrc.Response.Code)

	return
}

// Results -- struct returned by Crawl() to represent the entire crawl process
type Results struct {
	// Inherit the Resource struct
	Resource

	// Slice of Resource structs containing the needed resources for the given URL
	Resources []*Resource

	// ResourceTime shows how long it took to fetch all resources
	ResourceTime *TimerResult

	// TotalTime represents the time it took to crawl the site
	TotalTime *TimerResult
}

func (r *Results) String() string {
	if r.Resources != nil && r.ResourceTime != nil && r.TotalTime != nil {
		return fmt.Sprintf("<url(%s) == %d, resources(%d), resourceTime(%dms), totalTime(%dms), err(%s)>", r.URL, r.Response.Code, len(r.Resources), r.ResourceTime.Milli, r.TotalTime.Milli, r.Error)
	}

	return fmt.Sprintf("<url(%s), ip(%s), err(%s)>", r.request.URL, r.request.IP, r.Error)
}

var resourcePool sync.WaitGroup

// FetchURL manages the fetching of the main resource, as well as all child resources,
// providing a Results struct containing the entire crawl data needed
func (c *Crawler) FetchURL(URL string) (res *Results) {
	res = &Results{}
	crawlTimer := NewTimer()

	var err error

	res.request.URL = URL
	res.request.Host, err = getHost(URL)
	if err != nil {
		res.Error = err
		return
	}
	res.request.IP = c.ipmap[res.request.Host]

	// actually fetch the request
	resp, err := c.Get(URL)

	defer func() {
		crawlTimer.End()
		res.TotalTime = crawlTimer.Result
	}()

	if err != nil {
		res.Error = err
		return
	}

	defer resp.Body.Close()

	res.Response.Host = resp.Request.Host
	res.URL = resp.URL
	res.Response.Code = resp.StatusCode
	res.Response.Scheme = resp.Request.URL.Scheme
	res.Response.ContentLength = resp.ContentLength
	res.Response.TLS = resp.TLS
	res.Time = resp.Time

	if res.Response.Host != res.request.Host {
		res.Response.Remote = true
	}

	buf, _ := ioutil.ReadAll(resp.Body)
	b := ioutil.NopCloser(bytes.NewReader(buf))
	defer b.Close()

	bbytes, err := ioutil.ReadAll(bytes.NewBuffer(buf))
	if err == nil && len(bbytes) != 0 {
		res.Response.Body = string(bbytes[:])
	}

	urls := getSrc(b, resp.Request)

	c.Log.Printf("fetched %s in %dms with status %d", res.URL, res.Time.Milli, res.Response.Code)

	resourceTime := NewTimer()

	defer func() {
		resourceTime.End()
		res.ResourceTime = resourceTime.Result
	}()

	for i := range urls {
		resourcePool.Add(1)

		rsrc := &Resource{request: ResourceOrigin{URL: urls[i]}}
		res.Resources = append(res.Resources, rsrc)
		go c.fetchResource(res.Resources[i])
	}

	resourcePool.Wait()

	return
}

// Domain represents a url we need to fetch, including the items needed to
// fetch said url. E.g: host, port, ip, scheme, path, etc.
type Domain struct {
	URL *url.URL
	IP  string
}

// Crawler is the higher level struct which wraps the entire threaded crawl process
type Crawler struct {
	Log     *log.Logger
	Domains []*Domain
	Results []*Results
	ipmap   map[string]string
}

// Crawl represents the higher level functionality of scraper. Crawl should
// concurrently request the needed resources for a list of domains, allowing
// the bypass of DNS lookups where necessary.
func (c *Crawler) Crawl() {
	var results []*Results
	var wg sync.WaitGroup
	timer := NewTimer()

	// strip all common duplicate domain/ip pairs
	stripDups(&c.Domains)

	c.ipmap = make(map[string]string)
	var withwww string
	for i := range c.Domains {
		c.ipmap[c.Domains[i].URL.Host] = c.Domains[i].IP
		c.ipmap[strings.TrimPrefix(c.Domains[i].URL.Host, "www.")] = c.Domains[i].IP // no www. directive

		withwww = "www." + strings.TrimPrefix(c.Domains[i].URL.Host, "www.")
		c.ipmap[withwww] = c.Domains[i].IP // www. directive
	}

	// loop through all supplied urls and send them to a worker to be fetched
	for _, domain := range c.Domains {
		wg.Add(1)

		go func(domain *Domain) {
			defer wg.Done()

			result := c.FetchURL(domain.URL.String())
			results = append(results, result)

			if result.Error != nil {
				c.Log.Printf("error scanning %s (error: %s)", domain.URL.String(), result.Error)
			} else {
				c.Log.Printf("finished scanning %s (%dms)", domain.URL.String(), result.TotalTime.Milli)
			}
		}(domain)
	}

	// wait for all workers to complete their tasks
	wg.Wait()
	timer.End()

	c.Log.Printf("finished scanning %d urls in %d seconds", len(results), timer.Result.Seconds)

	// give some extra details
	var resSuccess, resError int
	for i := range results {
		if results[i].Error != nil {
			resError++
			continue
		}

		resSuccess++
	}

	c.Log.Printf("%d successful, %d errored", resSuccess, resError)

	c.Results = results

	return
}

// GetResults gets the potential results of a given requested url/ip
func (c *Crawler) GetResults(URL, IP string) *Results {
	for i := range c.Results {
		if c.Results[i].request.URL == URL && c.Results[i].request.IP == IP {
			return c.Results[i]
		}
	}

	return nil
}
<|endoftext|>