<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path"
	"strings"

	"github.com/davidrjonas/ssh-iam-bridge/strarray"
)

func check(err error, failureMessage string) {
	if err == nil {
		return
	}

	if exerr, ok := err.(*exec.ExitError); ok {
		os.Stderr.Write(exerr.Stderr)
	}

	fmt.Fprintln(os.Stderr, "!!!", failureMessage)
	os.Exit(1)
}

func backupFile(filename string) {
	fmt.Println("Backing up", filename, "to", filename+".orig")
	err := exec.Command("cp", "-f", filename, filename+".orig").Run()
	check(err, "Failed to backup "+filename)
}

func install(selfPath, username string) {
	cmdName := installAuthorizedKeysCommandScript(selfPath)
	installUser(username)
	installToSshd(cmdName, username)
	installToPam(selfPath)
	installToCron(selfPath)
}

// ssh is picky about AuthorizedKeysCommand, see man sshd_config
func installAuthorizedKeysCommandScript(selfPath string) string {
	cmdName := "/usr/sbin/ssh-iam-bridge-public-keys"
	fmt.Println("Writing AuthorizedKeysCommand script", cmdName)

	script := fmt.Sprintf("#!/bin/sh\nexec %s authorized_keys \"$@\"\n", selfPath)

	check(ioutil.WriteFile(cmdName, []byte(script), 0755), "Failed to write script"+cmdName)

	return cmdName
}

func installUser(username string) {
	_, err := user.Lookup(username)

	if err == nil {
		// User already exists
		return
	}

	if _, ok := err.(user.UnknownUserError); !ok {
		fmt.Fprintf(os.Stderr, "Failed to lookup user '%s'; %s", username, err)
		os.Exit(1)
	}

	fmt.Println("Creating SSH authorized keys lookup user", username)

	args := []string{
		"--system",
		"--shell", "/usr/sbin/nologin",
		"--comment", "SSH authorized keys lookup",
		username,
	}

	_, err = exec.Command("useradd", args...).Output()
	check(err, "Failed to create authorized keys lookup user")
}

func installToSshd(cmd, username string) {

	filename := "/etc/ssh/sshd_config"

	// TODO: Ensure 'PasswordAuthentication no' and 'UsePAM yes'

	linesToAdd := []string{
		"AuthorizedKeysCommand " + cmd + "\n",
		"AuthorizedKeysCommandUser " + username + "\n",
		"ChallengeResponseAuthentication yes\n",
		"AuthenticationMethods publickey keyboard-interactive:pam,publickey\n",
	}

	lines, err := strarray.ReadFile(filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to read '%s'; %s", filename, err)
		os.Exit(1)
	}

	if strarray.ContainsAll(lines, linesToAdd) {
		return
	}

	fmt.Println("Updating", filename)

	// Comment out specific lines
	linesToComment := []string{
		"AuthorizedKeysCommand ",
		"AuthorizedKeysCommandUser ",
		"ChallengeResponseAuthentication ",
		"AuthenticationMethods ",
	}

	for idx, line := range lines {
		for _, check := range linesToComment {
			if strings.HasPrefix(line, check) {
				lines[idx] = "# " + line
			}
		}
	}

	backupFile(filename)

	check(strarray.WriteFile(filename, lines, linesToAdd), "Failed to write changes to "+filename)

	err = exec.Command("sshd", "-t").Run()
	check(err, "Modified sshd_config failed to lint. Correct the errors before proceeding.")
}

func installToPam(selfPath string) {

	filename := "/etc/pam.d/sshd"
	fmt.Println("Updating", filename)

	pamExec := "auth requisite pamExec.so stdout quiet " + selfPath + " pam_create_user\n"

	lines, err := strarray.ReadFile(filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to read '%s'; %s", filename, err)
	}

	for _, line := range lines {
		if line == pamExec {
			return
		}
	}

	backupFile(filename)
	check(strarray.WriteFile(filename, []string{"# Next line added by " + selfPath + "\n", pamExec}, lines), "Failed to modify pam")
}

func installToCron(selfPath string) {

	filename := "/etc/cron.d/" + path.Base(selfPath)

	fmt.Println("Installing crontab", filename)

	contents := "*/10 * * * * root " + selfPath + " sync_groups"

	check(ioutil.WriteFile(filename, []byte(contents), 0644), "Failed to write "+filename)
}
<commit_msg>Revert camelCase mistake on pam_exec.so<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path"
	"strings"

	"github.com/davidrjonas/ssh-iam-bridge/strarray"
)

func check(err error, failureMessage string) {
	if err == nil {
		return
	}

	if exerr, ok := err.(*exec.ExitError); ok {
		os.Stderr.Write(exerr.Stderr)
	}

	fmt.Fprintln(os.Stderr, "!!!", failureMessage)
	os.Exit(1)
}

func backupFile(filename string) {
	fmt.Println("Backing up", filename, "to", filename+".orig")
	err := exec.Command("cp", "-f", filename, filename+".orig").Run()
	check(err, "Failed to backup "+filename)
}

func install(selfPath, username string) {
	cmdName := installAuthorizedKeysCommandScript(selfPath)
	installUser(username)
	installToSshd(cmdName, username)
	installToPam(selfPath)
	installToCron(selfPath)
}

// ssh is picky about AuthorizedKeysCommand, see man sshd_config
func installAuthorizedKeysCommandScript(selfPath string) string {
	cmdName := "/usr/sbin/ssh-iam-bridge-public-keys"
	fmt.Println("Writing AuthorizedKeysCommand script", cmdName)

	script := fmt.Sprintf("#!/bin/sh\nexec %s authorized_keys \"$@\"\n", selfPath)

	check(ioutil.WriteFile(cmdName, []byte(script), 0755), "Failed to write script"+cmdName)

	return cmdName
}

func installUser(username string) {
	_, err := user.Lookup(username)

	if err == nil {
		// User already exists
		return
	}

	if _, ok := err.(user.UnknownUserError); !ok {
		fmt.Fprintf(os.Stderr, "Failed to lookup user '%s'; %s", username, err)
		os.Exit(1)
	}

	fmt.Println("Creating SSH authorized keys lookup user", username)

	args := []string{
		"--system",
		"--shell", "/usr/sbin/nologin",
		"--comment", "SSH authorized keys lookup",
		username,
	}

	_, err = exec.Command("useradd", args...).Output()
	check(err, "Failed to create authorized keys lookup user")
}

func installToSshd(cmd, username string) {

	filename := "/etc/ssh/sshd_config"

	// TODO: Ensure 'PasswordAuthentication no' and 'UsePAM yes'

	linesToAdd := []string{
		"AuthorizedKeysCommand " + cmd + "\n",
		"AuthorizedKeysCommandUser " + username + "\n",
		"ChallengeResponseAuthentication yes\n",
		"AuthenticationMethods publickey keyboard-interactive:pam,publickey\n",
	}

	lines, err := strarray.ReadFile(filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to read '%s'; %s", filename, err)
		os.Exit(1)
	}

	if strarray.ContainsAll(lines, linesToAdd) {
		return
	}

	fmt.Println("Updating", filename)

	// Comment out specific lines
	linesToComment := []string{
		"AuthorizedKeysCommand ",
		"AuthorizedKeysCommandUser ",
		"ChallengeResponseAuthentication ",
		"AuthenticationMethods ",
	}

	for idx, line := range lines {
		for _, check := range linesToComment {
			if strings.HasPrefix(line, check) {
				lines[idx] = "# " + line
			}
		}
	}

	backupFile(filename)

	check(strarray.WriteFile(filename, lines, linesToAdd), "Failed to write changes to "+filename)

	err = exec.Command("sshd", "-t").Run()
	check(err, "Modified sshd_config failed to lint. Correct the errors before proceeding.")
}

func installToPam(selfPath string) {

	filename := "/etc/pam.d/sshd"
	fmt.Println("Updating", filename)

	pamExec := "auth requisite pam_exec.so stdout quiet " + selfPath + " pam_create_user\n"

	lines, err := strarray.ReadFile(filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to read '%s'; %s", filename, err)
	}

	for _, line := range lines {
		if line == pamExec {
			return
		}
	}

	backupFile(filename)
	check(strarray.WriteFile(filename, []string{"# Next line added by " + selfPath + "\n", pamExec}, lines), "Failed to modify pam")
}

func installToCron(selfPath string) {

	filename := "/etc/cron.d/" + path.Base(selfPath)

	fmt.Println("Installing crontab", filename)

	contents := "*/10 * * * * root " + selfPath + " sync_groups"

	check(ioutil.WriteFile(filename, []byte(contents), 0644), "Failed to write "+filename)
}
<|endoftext|>


<commit_before>package jsonapi

import (
	"net/http"
	"strconv"
	"strings"
)

// An Intent represents a valid combination of a request method and a URL pattern.
type Intent int

const (
	_ Intent = iota

	// ListResources is a variation of the following request:
	// GET /posts
	ListResources

	// FindResource is a variation of the following request:
	// GET /posts/1
	FindResource

	// CreateResource is a variation of the following request:
	// POST /posts
	CreateResource

	// UpdateResource is a variation of the following request:
	// PATCH /posts/1
	UpdateResource

	// DeleteResource is a variation of the following request:
	// DELETE /posts/1
	DeleteResource

	// GetRelatedResources is a variation of the following requests:
	// GET /posts/1/author
	// GET /posts/1/comments
	GetRelatedResources

	// GetRelationship is a variation of the following requests:
	// GET /posts/1/relationships/author
	// GET /posts/1/relationships/comments
	GetRelationship

	// SetRelationship is a variation of the following requests:
	// PATCH /posts/1/relationships/author.
	// PATCH /posts/1/relationships/comments.
	SetRelationship

	// AppendToRelationship is a variation of the following request:
	// POST /posts/1/relationships/comments
	AppendToRelationship

	// RemoveFromRelationship is a variation of the following request:
	// DELETE /posts/1/relationships/comments
	RemoveFromRelationship
)

// DocumentExpected returns whether a request using this intent is expected to
// include a JSON API document.
//
// Note: A response from an API may always include a document that at least
// contains one ore more errors.
func (i Intent) DocumentExpected() bool {
	switch i {
	case CreateResource, UpdateResource, SetRelationship,
		AppendToRelationship, RemoveFromRelationship:
		return true
	}

	return false
}

// RequestMethod returns the matching HTTP request method for an Intent.
func (i Intent) RequestMethod() string {
	switch i {
	case ListResources, FindResource, GetRelatedResources, GetRelationship:
		return "GET"
	case CreateResource, AppendToRelationship:
		return "POST"
	case UpdateResource, SetRelationship:
		return "PATCH"
	case DeleteResource, RemoveFromRelationship:
		return "DELETE"
	}

	return ""
}

// A Request contains all JSON API related information parsed from a low level
// request.
type Request struct {
	// The parsed JSON API intent of the request.
	Intent Intent

	// The prefix of the endpoint e.g. "api". It should not contain any prefix
	// or suffix slashes.
	Prefix string

	// The fragments parsed from the URL of the request. The fragments should not
	// contain any prefix or suffix slashes.
	ResourceType    string
	ResourceID      string
	RelatedResource string
	Relationship    string

	// The requested resources to be included in the response. This is read
	// from the "include" query parameter.
	Include []string

	// The pagination details of the request. Zero values mean no pagination
	// details have been provided. These values are read from the "page[number]",
	// "page[size]", "page[offset]" and "page[limit]" query parameters. These
	// parameters do not belong to the standard, but are recommended.
	PageNumber uint64
	PageSize   uint64
	PageOffset uint64
	PageLimit  uint64

	// The sorting that has been requested. This is read from the "sort" query
	// parameter.
	Sorting []string

	// The sparse fields that have been requested. This is read from the "fields"
	// query parameter.
	Fields map[string][]string

	// The filtering that has been requested. This is read from the "filter"
	// query parameter. This parameter does not belong to the standard, but is
	// recommended.
	Filters map[string][]string
}

// ParseRequest will parse the passed request and return a new Request with the
// parsed data. It will return an error if the content type, request method or
// url is invalid. Any returned error can directly be written using WriteError.
func ParseRequest(r *http.Request, prefix string) (*Request, error) {
	// get method
	method := r.Method

	// map method to action
	if method != "GET" && method != "POST" && method != "PATCH" && method != "DELETE" {
		return nil, BadRequest("Unsupported method")
	}

	// allocate new request
	jr := &Request{
		Prefix: strings.Trim(prefix, "/"),
	}

	// check content type header
	contentType := r.Header.Get("Content-Type")
	if contentType != "" && contentType != MediaType {
		return nil, BadRequest("Invalid content type header")
	}

	// check accept header
	accept := r.Header.Get("Accept")
	if accept != "" && accept != "*/*" && accept != "application/*" && accept != MediaType {
		return nil, ErrorFromStatus(http.StatusNotAcceptable, "Invalid accept header")
	}

	// de-prefix and trim path
	location := strings.TrimPrefix(strings.Trim(r.URL.Path, "/"), jr.Prefix+"/")

	// split path
	segments := strings.Split(location, "/")
	if len(segments) == 0 || len(segments) > 4 {
		return nil, BadRequest("Invalid URL segment count")
	}

	// check for invalid segments
	for _, s := range segments {
		if s == "" {
			return nil, BadRequest("Found empty URL segments")
		}
	}

	// set resource
	jr.ResourceType = segments[0]
	level := 1

	// set resource id
	if len(segments) > 1 {
		jr.ResourceID = segments[1]
		level = 2
	}

	// set related resource
	if len(segments) == 3 && segments[2] != "relationships" {
		jr.RelatedResource = segments[2]
		level = 3
	}

	// set relationship
	if len(segments) == 4 && segments[2] == "relationships" {
		jr.Relationship = segments[3]
		level = 4
	}

	// final check
	if len(segments) > 2 && (jr.RelatedResource == "" && jr.Relationship == "") {
		return nil, BadRequest("Invalid URL relationship format")
	}

	// calculate intent
	switch method {
	case "GET":
		switch level {
		case 1:
			jr.Intent = ListResources
		case 2:
			jr.Intent = FindResource
		case 3:
			jr.Intent = GetRelatedResources
		case 4:
			jr.Intent = GetRelationship
		}
	case "POST":
		switch level {
		case 1:
			jr.Intent = CreateResource
		case 4:
			jr.Intent = AppendToRelationship
		}
	case "PATCH":
		switch level {
		case 2:
			jr.Intent = UpdateResource
		case 4:
			jr.Intent = SetRelationship
		}
	case "DELETE":
		switch level {
		case 2:
			jr.Intent = DeleteResource
		case 4:
			jr.Intent = RemoveFromRelationship
		}
	}

	// check intent
	if jr.Intent == 0 {
		return nil, BadRequest("The URL and method combination is invalid")
	}

	// check if request should come with a document and has content type set
	if jr.Intent.DocumentExpected() && contentType == "" {
		return nil, BadRequest("Missing content type header")
	}

	for key, values := range r.URL.Query() {
		// set included resources
		if key == "include" {
			for _, v := range values {
				jr.Include = append(jr.Include, strings.Split(v, ",")...)
			}

			continue
		}

		// set sorting
		if key == "sort" {
			for _, v := range values {
				jr.Sorting = append(jr.Sorting, strings.Split(v, ",")...)
			}

			continue
		}

		// set page number
		if key == "page[number]" {
			if len(values) != 1 {
				return nil, BadRequestParam("More than one value", "page[number]")
			}

			n, err := strconv.ParseUint(values[0], 10, 0)
			if err != nil {
				return nil, BadRequestParam("Not a number", "page[number]")
			}

			jr.PageNumber = n
			continue
		}

		// set page size
		if key == "page[size]" {
			if len(values) != 1 {
				return nil, BadRequestParam("More than one value", "page[size]")
			}

			n, err := strconv.ParseUint(values[0], 10, 0)
			if err != nil {
				return nil, BadRequestParam("Not a number", "page[size]")
			}

			jr.PageSize = n
			continue
		}

		// set page offset
		if key == "page[offset]" {
			if len(values) != 1 {
				return nil, BadRequestParam("More than one value", "page[offset]")
			}

			n, err := strconv.ParseUint(values[0], 10, 0)
			if err != nil {
				return nil, BadRequestParam("Not a number", "page[offset]")
			}

			jr.PageOffset = n
			continue
		}

		// set page limit
		if key == "page[limit]" {
			if len(values) != 1 {
				return nil, BadRequestParam("More than one value", "page[limit]")
			}

			n, err := strconv.ParseUint(values[0], 10, 0)
			if err != nil {
				return nil, BadRequestParam("Not a number", "page[limit]")
			}

			jr.PageLimit = n
			continue
		}

		// set sparse fields
		if strings.HasPrefix(key, "fields[") && strings.HasSuffix(key, "]") {
			if jr.Fields == nil {
				jr.Fields = make(map[string][]string)
			}

			typ := key[7 : len(key)-1]

			for _, v := range values {
				jr.Fields[typ] = append(jr.Fields[typ], strings.Split(v, ",")...)
			}
		}

		// set filters
		if strings.HasPrefix(key, "filter[") && strings.HasSuffix(key, "]") {
			if jr.Filters == nil {
				jr.Filters = make(map[string][]string)
			}

			typ := key[7 : len(key)-1]

			for _, v := range values {
				jr.Filters[typ] = append(jr.Filters[typ], strings.Split(v, ",")...)
			}
		}
	}

	// check that page number is set if page size is set
	if jr.PageNumber > 0 && jr.PageSize <= 0 {
		return nil, BadRequestParam("Missing page size", "page[number]")
	}

	// check that page size is set if page number is set
	if jr.PageSize > 0 && jr.PageNumber <= 0 {
		return nil, BadRequestParam("Missing page number", "page[size]")
	}

	// check that page limit is set if page offset is set
	if jr.PageOffset > 0 && jr.PageLimit <= 0 {
		return nil, BadRequestParam("Missing page limit", "page[limit]")
	}

	return jr, nil
}

// Base will generate the base URL for this request, which includes the type and
// id if present.
func (r *Request) Base() string {
	// prepare segments
	var segments []string

	// add prefix if set
	if r.Prefix != "" {
		segments = append(segments, r.Prefix)
	}

	// add resource type
	segments = append(segments, r.ResourceType)

	// add id if available
	if r.ResourceID != "" {
		segments = append(segments, r.ResourceID)
	}

	return "/" + strings.Join(segments, "/")
}

// Self will generate the "self" URL for this request, which includes all path
// elements if available.
func (r *Request) Self() string {
	// prepare segments
	var segments []string

	// add prefix if set
	if r.Prefix != "" {
		segments = append(segments, r.Prefix)
	}

	// add resource type
	segments = append(segments, r.ResourceType)

	// add id if available
	if r.ResourceID != "" {
		segments = append(segments, r.ResourceID)

		// add related resource or relationship
		if r.RelatedResource != "" {
			segments = append(segments, r.RelatedResource)
		} else if r.Relationship != "" {
			segments = append(segments, "relationships", r.Relationship)
		}
	}

	return "/" + strings.Join(segments, "/")
}
<commit_msg>also allow "application/json" as accept header<commit_after>package jsonapi

import (
	"net/http"
	"strconv"
	"strings"
)

// An Intent represents a valid combination of a request method and a URL pattern.
type Intent int

const (
	_ Intent = iota

	// ListResources is a variation of the following request:
	// GET /posts
	ListResources

	// FindResource is a variation of the following request:
	// GET /posts/1
	FindResource

	// CreateResource is a variation of the following request:
	// POST /posts
	CreateResource

	// UpdateResource is a variation of the following request:
	// PATCH /posts/1
	UpdateResource

	// DeleteResource is a variation of the following request:
	// DELETE /posts/1
	DeleteResource

	// GetRelatedResources is a variation of the following requests:
	// GET /posts/1/author
	// GET /posts/1/comments
	GetRelatedResources

	// GetRelationship is a variation of the following requests:
	// GET /posts/1/relationships/author
	// GET /posts/1/relationships/comments
	GetRelationship

	// SetRelationship is a variation of the following requests:
	// PATCH /posts/1/relationships/author.
	// PATCH /posts/1/relationships/comments.
	SetRelationship

	// AppendToRelationship is a variation of the following request:
	// POST /posts/1/relationships/comments
	AppendToRelationship

	// RemoveFromRelationship is a variation of the following request:
	// DELETE /posts/1/relationships/comments
	RemoveFromRelationship
)

// DocumentExpected returns whether a request using this intent is expected to
// include a JSON API document.
//
// Note: A response from an API may always include a document that at least
// contains one ore more errors.
func (i Intent) DocumentExpected() bool {
	switch i {
	case CreateResource, UpdateResource, SetRelationship,
		AppendToRelationship, RemoveFromRelationship:
		return true
	}

	return false
}

// RequestMethod returns the matching HTTP request method for an Intent.
func (i Intent) RequestMethod() string {
	switch i {
	case ListResources, FindResource, GetRelatedResources, GetRelationship:
		return "GET"
	case CreateResource, AppendToRelationship:
		return "POST"
	case UpdateResource, SetRelationship:
		return "PATCH"
	case DeleteResource, RemoveFromRelationship:
		return "DELETE"
	}

	return ""
}

// A Request contains all JSON API related information parsed from a low level
// request.
type Request struct {
	// The parsed JSON API intent of the request.
	Intent Intent

	// The prefix of the endpoint e.g. "api". It should not contain any prefix
	// or suffix slashes.
	Prefix string

	// The fragments parsed from the URL of the request. The fragments should not
	// contain any prefix or suffix slashes.
	ResourceType    string
	ResourceID      string
	RelatedResource string
	Relationship    string

	// The requested resources to be included in the response. This is read
	// from the "include" query parameter.
	Include []string

	// The pagination details of the request. Zero values mean no pagination
	// details have been provided. These values are read from the "page[number]",
	// "page[size]", "page[offset]" and "page[limit]" query parameters. These
	// parameters do not belong to the standard, but are recommended.
	PageNumber uint64
	PageSize   uint64
	PageOffset uint64
	PageLimit  uint64

	// The sorting that has been requested. This is read from the "sort" query
	// parameter.
	Sorting []string

	// The sparse fields that have been requested. This is read from the "fields"
	// query parameter.
	Fields map[string][]string

	// The filtering that has been requested. This is read from the "filter"
	// query parameter. This parameter does not belong to the standard, but is
	// recommended.
	Filters map[string][]string
}

// ParseRequest will parse the passed request and return a new Request with the
// parsed data. It will return an error if the content type, request method or
// url is invalid. Any returned error can directly be written using WriteError.
func ParseRequest(r *http.Request, prefix string) (*Request, error) {
	// get method
	method := r.Method

	// map method to action
	if method != "GET" && method != "POST" && method != "PATCH" && method != "DELETE" {
		return nil, BadRequest("Unsupported method")
	}

	// allocate new request
	jr := &Request{
		Prefix: strings.Trim(prefix, "/"),
	}

	// check content type header
	contentType := r.Header.Get("Content-Type")
	if contentType != "" && contentType != MediaType {
		return nil, BadRequest("Invalid content type header")
	}

	// check accept header
	accept := r.Header.Get("Accept")
	if accept != "" && accept != "*/*" && accept != "application/*" && accept != "application/json" && accept != MediaType {
		return nil, ErrorFromStatus(http.StatusNotAcceptable, "Invalid accept header")
	}

	// de-prefix and trim path
	location := strings.TrimPrefix(strings.Trim(r.URL.Path, "/"), jr.Prefix+"/")

	// split path
	segments := strings.Split(location, "/")
	if len(segments) == 0 || len(segments) > 4 {
		return nil, BadRequest("Invalid URL segment count")
	}

	// check for invalid segments
	for _, s := range segments {
		if s == "" {
			return nil, BadRequest("Found empty URL segments")
		}
	}

	// set resource
	jr.ResourceType = segments[0]
	level := 1

	// set resource id
	if len(segments) > 1 {
		jr.ResourceID = segments[1]
		level = 2
	}

	// set related resource
	if len(segments) == 3 && segments[2] != "relationships" {
		jr.RelatedResource = segments[2]
		level = 3
	}

	// set relationship
	if len(segments) == 4 && segments[2] == "relationships" {
		jr.Relationship = segments[3]
		level = 4
	}

	// final check
	if len(segments) > 2 && (jr.RelatedResource == "" && jr.Relationship == "") {
		return nil, BadRequest("Invalid URL relationship format")
	}

	// calculate intent
	switch method {
	case "GET":
		switch level {
		case 1:
			jr.Intent = ListResources
		case 2:
			jr.Intent = FindResource
		case 3:
			jr.Intent = GetRelatedResources
		case 4:
			jr.Intent = GetRelationship
		}
	case "POST":
		switch level {
		case 1:
			jr.Intent = CreateResource
		case 4:
			jr.Intent = AppendToRelationship
		}
	case "PATCH":
		switch level {
		case 2:
			jr.Intent = UpdateResource
		case 4:
			jr.Intent = SetRelationship
		}
	case "DELETE":
		switch level {
		case 2:
			jr.Intent = DeleteResource
		case 4:
			jr.Intent = RemoveFromRelationship
		}
	}

	// check intent
	if jr.Intent == 0 {
		return nil, BadRequest("The URL and method combination is invalid")
	}

	// check if request should come with a document and has content type set
	if jr.Intent.DocumentExpected() && contentType == "" {
		return nil, BadRequest("Missing content type header")
	}

	for key, values := range r.URL.Query() {
		// set included resources
		if key == "include" {
			for _, v := range values {
				jr.Include = append(jr.Include, strings.Split(v, ",")...)
			}

			continue
		}

		// set sorting
		if key == "sort" {
			for _, v := range values {
				jr.Sorting = append(jr.Sorting, strings.Split(v, ",")...)
			}

			continue
		}

		// set page number
		if key == "page[number]" {
			if len(values) != 1 {
				return nil, BadRequestParam("More than one value", "page[number]")
			}

			n, err := strconv.ParseUint(values[0], 10, 0)
			if err != nil {
				return nil, BadRequestParam("Not a number", "page[number]")
			}

			jr.PageNumber = n
			continue
		}

		// set page size
		if key == "page[size]" {
			if len(values) != 1 {
				return nil, BadRequestParam("More than one value", "page[size]")
			}

			n, err := strconv.ParseUint(values[0], 10, 0)
			if err != nil {
				return nil, BadRequestParam("Not a number", "page[size]")
			}

			jr.PageSize = n
			continue
		}

		// set page offset
		if key == "page[offset]" {
			if len(values) != 1 {
				return nil, BadRequestParam("More than one value", "page[offset]")
			}

			n, err := strconv.ParseUint(values[0], 10, 0)
			if err != nil {
				return nil, BadRequestParam("Not a number", "page[offset]")
			}

			jr.PageOffset = n
			continue
		}

		// set page limit
		if key == "page[limit]" {
			if len(values) != 1 {
				return nil, BadRequestParam("More than one value", "page[limit]")
			}

			n, err := strconv.ParseUint(values[0], 10, 0)
			if err != nil {
				return nil, BadRequestParam("Not a number", "page[limit]")
			}

			jr.PageLimit = n
			continue
		}

		// set sparse fields
		if strings.HasPrefix(key, "fields[") && strings.HasSuffix(key, "]") {
			if jr.Fields == nil {
				jr.Fields = make(map[string][]string)
			}

			typ := key[7 : len(key)-1]

			for _, v := range values {
				jr.Fields[typ] = append(jr.Fields[typ], strings.Split(v, ",")...)
			}
		}

		// set filters
		if strings.HasPrefix(key, "filter[") && strings.HasSuffix(key, "]") {
			if jr.Filters == nil {
				jr.Filters = make(map[string][]string)
			}

			typ := key[7 : len(key)-1]

			for _, v := range values {
				jr.Filters[typ] = append(jr.Filters[typ], strings.Split(v, ",")...)
			}
		}
	}

	// check that page number is set if page size is set
	if jr.PageNumber > 0 && jr.PageSize <= 0 {
		return nil, BadRequestParam("Missing page size", "page[number]")
	}

	// check that page size is set if page number is set
	if jr.PageSize > 0 && jr.PageNumber <= 0 {
		return nil, BadRequestParam("Missing page number", "page[size]")
	}

	// check that page limit is set if page offset is set
	if jr.PageOffset > 0 && jr.PageLimit <= 0 {
		return nil, BadRequestParam("Missing page limit", "page[limit]")
	}

	return jr, nil
}

// Base will generate the base URL for this request, which includes the type and
// id if present.
func (r *Request) Base() string {
	// prepare segments
	var segments []string

	// add prefix if set
	if r.Prefix != "" {
		segments = append(segments, r.Prefix)
	}

	// add resource type
	segments = append(segments, r.ResourceType)

	// add id if available
	if r.ResourceID != "" {
		segments = append(segments, r.ResourceID)
	}

	return "/" + strings.Join(segments, "/")
}

// Self will generate the "self" URL for this request, which includes all path
// elements if available.
func (r *Request) Self() string {
	// prepare segments
	var segments []string

	// add prefix if set
	if r.Prefix != "" {
		segments = append(segments, r.Prefix)
	}

	// add resource type
	segments = append(segments, r.ResourceType)

	// add id if available
	if r.ResourceID != "" {
		segments = append(segments, r.ResourceID)

		// add related resource or relationship
		if r.RelatedResource != "" {
			segments = append(segments, r.RelatedResource)
		} else if r.Relationship != "" {
			segments = append(segments, "relationships", r.Relationship)
		}
	}

	return "/" + strings.Join(segments, "/")
}
<|endoftext|>


<commit_before>package goinsta

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

type reqOptions struct {
	// Endpoint is the request path of instagram api
	Endpoint string

	// IsPost setted to true will send request with POST method.
	//
	// By default this option is false.
	IsPost bool

	// Query is the parameters of the request
	//
	// This parameters are independents of the request method (POST|GET)
	Query map[string]string
}

func (insta *Instagram) sendSimpleRequest(uri string, a ...interface{}) (body []byte, err error) {
	return insta.sendRequest(&reqOptions{
		Endpoint: fmt.Sprintf(uri, a...),
	})
}

func (inst *Instagram) sendRequest(o *reqOptions) (body []byte, err error) {
	if !inst.logged {
		return nil, ErrLoggedOut
	}

	method := "GET"
	if o.IsPost {
		method = "POST"
	}

	u, err := url.Parse(goInstaAPIUrl + o.Endpoint)
	if err != nil {
		return nil, err
	}

	bf := bytes.NewBuffer([]byte{})

	q := u.Query()
	for k, v := range o.Query {
		q.Add(k, v)
	}

	if o.IsPost {
		bf.WriteString(q.Encode())
	} else {
		u.RawQuery = q.Encode()
	}

	var req *http.Request
	req, err = http.NewRequest(method, u.String(), bf)
	if err != nil {
		return
	}

	req.Header.Set("Connection", "close")
	req.Header.Set("Accept", "*/*")
	req.Header.Set("Content-type", "application/x-www-form-urlencoded; charset=UTF-8")
	req.Header.Set("Cookie2", "$Version=1")
	req.Header.Set("Accept-Language", "en-US")
	req.Header.Set("User-Agent", goInstaUserAgent)

	resp, err := inst.c.Do(req)
	if err != nil {
		return body, err
	}
	defer resp.Body.Close()

	u, _ = url.Parse(goInstaAPIUrl)
	for _, value := range inst.c.Jar.Cookies(u) {
		if strings.Contains(value.Name, "csrftoken") {
			inst.token = value.Value
		}
	}

	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	if resp.StatusCode != 200 {
		e := fmt.Errorf("Invalid status code %s", string(body))
		switch resp.StatusCode {
		case 400:
			e = ErrLoggedOut
		case 404:
			e = ErrNotFound
		}
		return nil, e
	}

	return body, err
}

func (insta *Instagram) prepareData(otherData ...map[string]interface{}) (string, error) {
	data := map[string]interface{}{
		"_uuid": insta.uuid,
		//"_uid":       insta.CurrentUser.ID,
		"_csrftoken": insta.token,
	}
	for i := range otherData {
		for key, value := range otherData[i] {
			data[key] = value
		}
	}
	bytes, err := json.Marshal(data)
	return string(bytes), err
}
<commit_msg>Changed request functions<commit_after>package goinsta

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

type reqOptions struct {
	// Endpoint is the request path of instagram api
	Endpoint string

	// IsPost setted to true will send request with POST method.
	//
	// By default this option is false.
	IsPost bool

	// Query is the parameters of the request
	//
	// This parameters are independents of the request method (POST|GET)
	Query map[string]string
}

func (insta *Instagram) sendSimpleRequest(uri string, a ...interface{}) (body []byte, err error) {
	return insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(uri, a...),
		},
	)
}

func (inst *Instagram) sendRequest(o *reqOptions) (body []byte, err error) {
	if !inst.logged {
		return nil, ErrLoggedOut
	}

	method := "GET"
	if o.IsPost {
		method = "POST"
	}

	u, err := url.Parse(goInstaAPIUrl + o.Endpoint)
	if err != nil {
		return nil, err
	}

	bf := bytes.NewBuffer([]byte{})

	q := u.Query()
	for k, v := range o.Query {
		q.Add(k, v)
	}

	if o.IsPost {
		bf.WriteString(q.Encode())
	} else {
		u.RawQuery = q.Encode()
	}

	var req *http.Request
	req, err = http.NewRequest(method, u.String(), bf)
	if err != nil {
		return
	}

	req.Header.Set("Connection", "close")
	req.Header.Set("Accept", "*/*")
	req.Header.Set("Content-type", "application/x-www-form-urlencoded; charset=UTF-8")
	req.Header.Set("Cookie2", "$Version=1")
	req.Header.Set("Accept-Language", "en-US")
	req.Header.Set("User-Agent", goInstaUserAgent)

	resp, err := inst.c.Do(req)
	if err != nil {
		return body, err
	}
	defer resp.Body.Close()

	u, _ = url.Parse(goInstaAPIUrl)
	for _, value := range inst.c.Jar.Cookies(u) {
		if strings.Contains(value.Name, "csrftoken") {
			inst.token = value.Value
		}
	}

	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	if resp.StatusCode != 200 {
		switch resp.StatusCode {
		case 400:
			err = ErrLoggedOut
		case 404:
			err = ErrNotFound
		default:
			err = fmt.Errorf("Invalid status code %s", string(body))
		}
	}

	return body, err
}

func (insta *Instagram) prepareData(other ...map[string]interface{}) ([]byte, error) {
	data := map[string]interface{}{
		"_uuid":      insta.uuid,
		"_uid":       insta.Account.ID,
		"_csrftoken": insta.token,
	}
	for i := range other {
		for key, value := range other[i] {
			data[key] = value
		}
	}
	return json.Marshal(data)
}
<|endoftext|>


<commit_before>package guber

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Request struct {
	client    *Client
	method    string
	baseurl   string
	query     string
	resource  string
	namespace string
	name      string
	body      []byte

	// NOTE this is used distinct from err, because a 404 is not technically an
	// error, except to the end-user who expects a resource to be there.
	// Without this, we don't have a way to determine if an err was a 404 or
	// something lower-level without inspecting the error message.
	found bool

	err      error
	response *http.Response
}

func (r *Request) error(err error) {
	if err != nil && r.err == nil {
		r.err = err
	}
}

func (r *Request) url() string {
	path := ""
	if r.namespace != "" {
		path = fmt.Sprintf("namespaces/%s/", r.namespace)
	}
	path = path + r.resource
	if r.name != "" {
		path = path + "/" + r.name
	}
	if r.query != "" {
		path = path + "?" + r.query
	}
	return r.baseurl + "/" + path
}

func (r *Request) Resource(res Resource) *Request {
	baseurl := fmt.Sprintf("https://%s", r.client.Host)
	if res.DomainName() != "" {
		baseurl = fmt.Sprintf("%s/%s", baseurl, res.DomainName())
	}
	r.baseurl = fmt.Sprintf("%s/%s/%s", baseurl, res.APIGroup(), res.APIVersion())
	r.resource = res.APIName()
	return r
}

func (r *Request) Namespace(namespace string) *Request {
	r.namespace = namespace
	return r
}

func (r *Request) Name(name string) *Request {
	r.name = name
	return r
}

func (r *Request) Entity(e Entity) *Request {
	body, err := json.Marshal(e)
	r.body = body
	r.error(err)
	return r
}

func (r *Request) Query(q *QueryParams) *Request {
	if q == nil {
		return r
	}

	// v, err := query.Values(q)
	// if err != nil {
	// 	panic(err) // TODO should use r.error() here probably
	// }
	// queryStr := v.Encode()

	// TODO  -- we went with this terribly rigid strategy because of how query pkg encodes the = chars
	if ls := q.LabelSelector; ls != "" {
		r.query = "labelSelector=" + ls
	}

	return r
}

func (r *Request) Do() *Request {
	req, err := http.NewRequest(r.method, r.url(), bytes.NewBuffer(r.body))
	if err != nil {
		panic(err) // TODO
	}

	req.SetBasicAuth(r.client.Username, r.client.Password)
	r.error(err)

	resp, err := r.client.http.Do(req)
	r.error(err)

	// TODO
	if resp != nil {
		r.response = resp

		r.readBody()

		if resp.StatusCode == 404 {
			r.found = false
		} else if status := resp.Status; status[:2] != "20" {
			errMsg := fmt.Sprintf("Status: %s, Body: %s", status, string(r.body))
			r.error(errors.New(errMsg))
			r.found = false
		} else {
			r.found = true // NOTE this only really matters for lookups, but we set it true here anyhow
		}
	}
	return r
}

func (r *Request) readBody() {
	if r.response == nil {
		r.error(errors.New("Response is nil"))
		return
	}
	defer r.response.Body.Close()
	body, err := ioutil.ReadAll(r.response.Body)
	r.body = body
	r.error(err)
}

// The exit point for a Request (where error is pooped out)
func (r *Request) Into(e Entity) error {
	if r.body != nil {
		json.Unmarshal(r.body, e)
	}
	return r.err
}
<commit_msg>Use fmt.Errorf for error message on failed request<commit_after>package guber

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Request struct {
	client    *Client
	method    string
	baseurl   string
	query     string
	resource  string
	namespace string
	name      string
	body      []byte

	// NOTE this is used distinct from err, because a 404 is not technically an
	// error, except to the end-user who expects a resource to be there.
	// Without this, we don't have a way to determine if an err was a 404 or
	// something lower-level without inspecting the error message.
	found bool

	err      error
	response *http.Response
}

func (r *Request) error(err error) {
	if err != nil && r.err == nil {
		r.err = err
	}
}

func (r *Request) url() string {
	path := ""
	if r.namespace != "" {
		path = fmt.Sprintf("namespaces/%s/", r.namespace)
	}
	path = path + r.resource
	if r.name != "" {
		path = path + "/" + r.name
	}
	if r.query != "" {
		path = path + "?" + r.query
	}
	return r.baseurl + "/" + path
}

func (r *Request) Resource(res Resource) *Request {
	baseurl := fmt.Sprintf("https://%s", r.client.Host)
	if res.DomainName() != "" {
		baseurl = fmt.Sprintf("%s/%s", baseurl, res.DomainName())
	}
	r.baseurl = fmt.Sprintf("%s/%s/%s", baseurl, res.APIGroup(), res.APIVersion())
	r.resource = res.APIName()
	return r
}

func (r *Request) Namespace(namespace string) *Request {
	r.namespace = namespace
	return r
}

func (r *Request) Name(name string) *Request {
	r.name = name
	return r
}

func (r *Request) Entity(e Entity) *Request {
	body, err := json.Marshal(e)
	r.body = body
	r.error(err)
	return r
}

func (r *Request) Query(q *QueryParams) *Request {
	if q == nil {
		return r
	}

	// v, err := query.Values(q)
	// if err != nil {
	// 	panic(err) // TODO should use r.error() here probably
	// }
	// queryStr := v.Encode()

	// TODO  -- we went with this terribly rigid strategy because of how query pkg encodes the = chars
	if ls := q.LabelSelector; ls != "" {
		r.query = "labelSelector=" + ls
	}

	return r
}

func (r *Request) Do() *Request {
	req, err := http.NewRequest(r.method, r.url(), bytes.NewBuffer(r.body))
	if err != nil {
		panic(err) // TODO
	}

	req.SetBasicAuth(r.client.Username, r.client.Password)
	r.error(err)

	resp, err := r.client.http.Do(req)
	r.error(err)

	// TODO
	if resp != nil {
		r.response = resp

		r.readBody()

		if resp.StatusCode == 404 {
			r.found = false
		} else if status := resp.Status; status[:2] != "20" {
			r.error(fmt.Errorf("Status: %s, Body: %s", status, string(r.body)))
			r.found = false
		} else {
			r.found = true // NOTE this only really matters for lookups, but we set it true here anyhow
		}
	}
	return r
}

func (r *Request) readBody() {
	if r.response == nil {
		r.error(errors.New("Response is nil"))
		return
	}
	defer r.response.Body.Close()
	body, err := ioutil.ReadAll(r.response.Body)
	r.body = body
	r.error(err)
}

// The exit point for a Request (where error is pooped out)
func (r *Request) Into(e Entity) error {
	if r.body != nil {
		json.Unmarshal(r.body, e)
	}
	return r.err
}
<|endoftext|>


<commit_before>// Package som provides experimental implementation of Self-Organizing Map.
// See https://en.wikipedia.org/wiki/Self-organizing_map.
//
// SOM - Self-Organizing Map
// BMU - Best Matching Unit
package som

import (
	"errors"
	"math"
	"math/rand"
)

var (
	// ErrNoDataLeft is returned by selector when there is
	// nothing to select from the corresponding data set.
	ErrNoDataLeft = errors.New("no data left")
)

// RestraintFunc calculates learning restraint coefficient
// based on current iteration and overall iterations number.
// The coefficient value indicates how much neurons weights
// will change at corresponding iteration.
type RestraintFunc interface {
	// currentIt => [0, iterationsNumber)
	Apply(currentIt, iterationsNumber int) float64
}

// InfluenceFunc calculates the coefficient which indicates how much
// the weights of each neuron will be changed according to the BMU position.
type InfluenceFunc interface {
	// currentIt => [0, iterationsNumber)
	Apply(bmu *Neuron, currentIt, iterationsNumber, i, j int) float64
}

// DistanceFunc calculates Distance between two points
// represented as float vectors.
type DistanceFunc interface {
	Apply(xVector, yVector []float64) float64
}

// Selector is an interface for selecting data vectors
// from the data set in implementation specific manner.
type Selector interface {
	// Init initializes this Selector with the data set.
	Init(set *DataSet)

	// Next returns the next data vector from the data set,
	// or an error if there are no data vectors left.
	Next() (DataVector, error)
}

// NeuronsInitializer initializes neurons, for example sets
// initial values of weights. Called within Learn func before anything else.
type NeuronsInitializer interface {
	Init(set *DataSet, neurons [][]*Neuron)
}

// Neuron is a build unit in SOM.
// One neuron manages number of weights equal to the number of input vector elements(data set width).
// Each neuron is indexed and has its unique place in a map.
type Neuron struct {
	Weights  []float64
	Distance float64
	X, Y     int
}

// New creates new 2 dimensional X*Y size SOM.
func New(X, Y int) *SOM {
	neurons := make([][]*Neuron, X)
	for i := 0; i < X; i++ {
		neurons[i] = make([]*Neuron, Y)
	}

	for i := 0; i < X; i++ {
		for j := 0; j < Y; j++ {
			neurons[i][j] = &Neuron{X: i, Y: j}
		}
	}

	return &SOM{
		Neurons:     neurons,
		Initializer: &ZeroValueWeightsInitializer{},
		Selector:    &SequentialSelector{},
		Restraint:   &NoRestraintFunc{},
		Influence:   &BMUOnlyInfluencedFunc{},
		Distance:    &EuclideanDistanceFunc{},
	}
}

// SOM is a map itself.
// Currently it carries double dimension array of neurons,
// provides ability to teach the map and then use results.
type SOM struct {
	Neurons [][]*Neuron

	Initializer NeuronsInitializer
	Selector    Selector
	Restraint   RestraintFunc
	Influence   InfluenceFunc
	Distance    DistanceFunc
}

func (som *SOM) Learn(set *DataSet, iterationsNumber int) {
	som.Initializer.Init(set, som.Neurons)
	som.Selector.Init(set)
	for it := 0; it < iterationsNumber; it++ {
		vector, err := som.Selector.Next()
		if err != nil {
			break
		}

		som.computeDistance(vector)
		bmu := som.findBMU()
		som.fixWeights(it, iterationsNumber, bmu, vector)
	}
}

func (som *SOM) Test(vector DataVector) *Neuron {
	som.computeDistance(vector)
	return som.findBMU()
}

func (som *SOM) computeDistance(vector DataVector) {
	for i := 0; i < len(som.Neurons); i++ {
		for j := 0; j < len(som.Neurons[i]); j++ {
			som.Neurons[i][j].Distance = som.Distance.Apply(vector, som.Neurons[i][j].Weights)
		}
	}
}

func (som *SOM) findBMU() *Neuron {
	bmu := som.Neurons[0][0]
	minDistance := bmu.Distance
	candidatesCount := 1
	for i := 0; i < len(som.Neurons); i++ {
		for j := 0; j < len(som.Neurons[i]); j++ {
			candidate := som.Neurons[i][j]
			if minDistance > candidate.Distance {
				bmu = candidate
				minDistance = bmu.Distance
				candidatesCount = 1
			} else if minDistance == candidate.Distance {
				candidatesCount++
			}
		}
	}

	if candidatesCount == 1 {
		return bmu
	}

	candidates := make([]*Neuron, 0, 2)
	for i := 0; i < len(som.Neurons); i++ {
		for j := 0; j < len(som.Neurons[i]); j++ {
			if minDistance == som.Neurons[i][j].Distance {
				candidates = append(candidates, som.Neurons[i][j])
			}
		}
	}

	return candidates[rand.Intn(len(candidates))]
}

func (som *SOM) fixWeights(t, T int, bmu *Neuron, input DataVector) {
	for i := 0; i < len(som.Neurons); i++ {
		for j := 0; j < len(som.Neurons[i]); j++ {
			neuron := som.Neurons[i][j]
			for k := 0; k < len(neuron.Weights); k++ {
				cof := som.Restraint.Apply(t, T) * som.Influence.Apply(bmu, t, T, i, j)
				neuron.Weights[k] += cof * (input[k] - neuron.Weights[k])
			}
		}
	}
}

type EuclideanDistanceFunc struct{}

func (ed *EuclideanDistanceFunc) Apply(xVector, yVector []float64) float64 {
	var sum float64
	for i := 0; i < len(xVector); i++ {
		sum += math.Pow(xVector[i]-yVector[i], 2)
	}
	return math.Sqrt(sum)
}

// BMUOnlyInfluencedFunc is implementation of InfluenceFunc which
// allows modification of BMU neuron only.
type BMUOnlyInfluencedFunc struct{}

func (calc *BMUOnlyInfluencedFunc) Apply(bmu *Neuron, currentIt, iterationsNumber, i, j int) float64 {
	if bmu.X == i && bmu.Y == j {
		return 1
	} else {
		return 0
	}
}

// NoRestraintFunc is RestraintFunc implementation which always returns 1,
// thus doesn't effect weights modification at all.
type NoRestraintFunc struct{}

func (rc *NoRestraintFunc) Apply(currentIt, iterationsNumber int) float64 { return 1 }

type SequentialSelector struct {
	set *DataSet
	idx int
}

func (sel *SequentialSelector) Init(set *DataSet) {
	sel.set = set
}

func (sel *SequentialSelector) Next() (DataVector, error) {
	if sel.idx >= sel.set.Len() {
		return nil, ErrNoDataLeft
	}
	vector := sel.set.Vectors[sel.idx]
	sel.idx++
	return vector, nil
}

// RandSelector randomly selects a data vector from the corresponding data set,
// the selection is infinite, thus Next() never returns error. If data set size is X
// then X calls to Next() will return X different random vectors from the data set.
type RandSelector struct {
	dataSet *DataSet
	perm    []int
	idx     int
}

func (sel *RandSelector) Init(dataSet *DataSet) {
	sel.dataSet = dataSet
	sel.perm = rand.Perm(dataSet.Len())
}

func (sel *RandSelector) Next() (DataVector, error) {
	if sel.idx == len(sel.perm) {
		sel.idx = 0
		sel.perm = rand.Perm(sel.dataSet.Len())
	}
	vector := sel.dataSet.Vectors[sel.perm[sel.idx]]
	sel.idx++
	return vector, nil
}

// ZeroValueWeightsInitializer adjusts weight arrays length based on data set width.
type ZeroValueWeightsInitializer struct{}

func (initializer *ZeroValueWeightsInitializer) Init(set *DataSet, neurons [][]*Neuron) {
	inputSize := set.Width()
	for i := 0; i < len(neurons); i++ {
		for j := 0; j < len(neurons[i]); j++ {
			neurons[i][j].Weights = make([]float64, inputSize)
		}
	}
}

// RandWeightsInitializer sets weights values to small [0.0,1.0) random values.
type RandWeightsInitializer struct{}

func (initializer *RandWeightsInitializer) Init(set *DataSet, neurons [][]*Neuron) {
	zeroInitializer := &ZeroValueWeightsInitializer{}
	zeroInitializer.Init(set, neurons)

	for i := 0; i < len(neurons); i++ {
		for j := 0; j < len(neurons[i]); j++ {
			neuron := neurons[i][j]
			for k := 0; k < len(neuron.Weights); k++ {
				neuron.Weights[k] = rand.Float64()
			}
		}
	}
}

// RandDataSetVectorsWeightsInitializer sets weights values to random vectors from data set.
type RandDataSetVectorsWeightsInitializer struct{}

func (initializer *RandDataSetVectorsWeightsInitializer) Init(dataSet *DataSet, neurons [][]*Neuron) {
	zeroInitializer := &ZeroValueWeightsInitializer{}
	zeroInitializer.Init(dataSet, neurons)

	matrixSize := len(neurons) * len(neurons[0])
	if matrixSize < dataSet.Len() {
		dataSet = dataSet.Copy()
		dataSet.Sort()
		dataSet.Reduce(matrixSize)
	}

	selector := &RandSelector{}
	selector.Init(dataSet)

	for i := 0; i < len(neurons); i++ {
		for j := 0; j < len(neurons[i]); j++ {
			neuron := neurons[i][j]
			vector, _ := selector.Next()
			for k := 0; k < len(neuron.Weights); k++ {
				neuron.Weights[k] = vector[k]
			}
		}
	}
}

// RadiusReducingConstantInfluenceFunc influences only neurons in a given radius around BMU.
// Radius is reduced at each iteration, so the influence area becomes smaller,
// but not smaller than r/2, so R >= influence area > R/2.
type RadiusReducingConstantInfluenceFunc struct {
	Radius float64
}

func (influence *RadiusReducingConstantInfluenceFunc) Apply(bmu *Neuron, currentIt, iterationsNumber, i, j int) float64 {
	t := float64(currentIt)
	T := float64(iterationsNumber)
	qt := influence.Radius / (1 + t/T)

	d := math.Sqrt(math.Pow(float64(bmu.X-i), 2) + math.Pow(float64(bmu.Y-j), 2))

	if d > qt {
		return 0
	} else {
		return 1
	}
}

// GaussianInfluenceFunc calculates influence coefficient g(t) in the following way:
// g(t) = exp( -d**2/ (2*q(t)**2) )
// where q(T) is neighbourhood width function q(t) = InitialWidth * exp(-t/T),
// where d is euclidean distance from the BMU to [i][j] neuron
type GaussianInfluenceFunc struct {
	InitialWidth float64
}

func (gif *GaussianInfluenceFunc) Apply(bmu *Neuron, currentIt, iterationsNumber, i, j int) float64 {
	t := float64(currentIt)
	T := float64(iterationsNumber)
	q := gif.InitialWidth * math.Exp(-t/T)
	d := math.Sqrt(math.Pow(float64(bmu.X-i), 2) + math.Pow(float64(bmu.Y-j), 2))
	return math.Exp(-(d * d) / (2 * q * q))
}

// SimpleRestraintFunc calculates coefficient as => A / (B + t).
type SimpleRestraintFunc struct {
	A, B float64
}

func (rc *SimpleRestraintFunc) Apply(currentIt, iterationsNumber int) float64 {
	return rc.A / (rc.B + float64(currentIt))
}

// ExpRestraintFunc calculates coefficient as => InitialRate * exp(-t/T)
type ExpRestraintFunc struct {
	InitialRate float64
}

func (erf *ExpRestraintFunc) Apply(currentIt, iterationsNumber int) float64 {
	t := float64(currentIt)
	T := float64(iterationsNumber)
	return erf.InitialRate * math.Exp(-t/T)
}
<commit_msg>Add ability to monitor learning progress<commit_after>// Package som provides experimental implementation of Self-Organizing Map.
// See https://en.wikipedia.org/wiki/Self-organizing_map.
//
// SOM - Self-Organizing Map
// BMU - Best Matching Unit
package som

import (
	"errors"
	"math"
	"math/rand"
)

var (
	// ErrNoDataLeft is returned by selector when there is
	// nothing to select from the corresponding data set.
	ErrNoDataLeft = errors.New("no data left")
)

// RestraintFunc calculates learning restraint coefficient
// based on current iteration and overall iterations number.
// The coefficient value indicates how much neurons weights
// will change at corresponding iteration.
type RestraintFunc interface {
	// currentIt => [0, iterationsNumber)
	Apply(currentIt, iterationsNumber int) float64
}

// InfluenceFunc calculates the coefficient which indicates how much
// the weights of each neuron will be changed according to the BMU position.
type InfluenceFunc interface {
	// currentIt => [0, iterationsNumber)
	Apply(bmu *Neuron, currentIt, iterationsNumber, i, j int) float64
}

// DistanceFunc calculates Distance between two points
// represented as float vectors.
type DistanceFunc interface {
	Apply(xVector, yVector []float64) float64
}

// Selector is an interface for selecting data vectors
// from the data set in implementation specific manner.
type Selector interface {
	// Init initializes this Selector with the data set.
	Init(set *DataSet)

	// Next returns the next data vector from the data set,
	// or an error if there are no data vectors left.
	Next() (DataVector, error)
}

// NeuronsInitializer initializes neurons, for example sets
// initial values of weights. Called within Learn func before anything else.
type NeuronsInitializer interface {
	Init(set *DataSet, neurons [][]*Neuron)
}

// ProgressMonitor handler for monitoring learning progress.
type ProgressMonitor interface {
	// ItCompleted called each time iteration completes.
	// Iteration is within bounds [1, itNum].
	ItCompleted(it, itNum int, som *SOM)
}

// Neuron is a build unit in SOM.
// One neuron manages number of weights equal to the number of input vector elements(data set width).
// Each neuron is indexed and has its unique place in a map.
type Neuron struct {
	Weights  []float64
	Distance float64
	X, Y     int
}

// New creates new 2 dimensional X*Y size SOM.
func New(X, Y int) *SOM {
	neurons := make([][]*Neuron, X)
	for i := 0; i < X; i++ {
		neurons[i] = make([]*Neuron, Y)
	}

	for i := 0; i < X; i++ {
		for j := 0; j < Y; j++ {
			neurons[i][j] = &Neuron{X: i, Y: j}
		}
	}

	return &SOM{
		Neurons:     neurons,
		Initializer: &ZeroValueWeightsInitializer{},
		Selector:    &SequentialSelector{},
		Restraint:   &NoRestraintFunc{},
		Influence:   &BMUOnlyInfluencedFunc{},
		Distance:    &EuclideanDistanceFunc{},
		Monitor:     &NoOpProgressMonitor{},
	}
}

// SOM is a map itself.
// Currently it carries double dimension array of neurons,
// provides ability to teach the map and then use results.
type SOM struct {
	Neurons [][]*Neuron

	Initializer NeuronsInitializer
	Selector    Selector
	Restraint   RestraintFunc
	Influence   InfluenceFunc
	Distance    DistanceFunc
	Monitor     ProgressMonitor
}

func (som *SOM) Learn(set *DataSet, iterationsNumber int) {
	som.Initializer.Init(set, som.Neurons)
	som.Selector.Init(set)
	for it := 0; it < iterationsNumber; it++ {
		vector, err := som.Selector.Next()
		if err != nil {
			break
		}

		som.computeDistance(vector)
		bmu := som.findBMU()
		som.fixWeights(it, iterationsNumber, bmu, vector)

		som.Monitor.ItCompleted(it+1, iterationsNumber, som)
	}
}

func (som *SOM) Test(vector DataVector) *Neuron {
	som.computeDistance(vector)
	return som.findBMU()
}

func (som *SOM) computeDistance(vector DataVector) {
	for i := 0; i < len(som.Neurons); i++ {
		for j := 0; j < len(som.Neurons[i]); j++ {
			som.Neurons[i][j].Distance = som.Distance.Apply(vector, som.Neurons[i][j].Weights)
		}
	}
}

func (som *SOM) findBMU() *Neuron {
	bmu := som.Neurons[0][0]
	minDistance := bmu.Distance
	candidatesCount := 1
	for i := 0; i < len(som.Neurons); i++ {
		for j := 0; j < len(som.Neurons[i]); j++ {
			candidate := som.Neurons[i][j]
			if minDistance > candidate.Distance {
				bmu = candidate
				minDistance = bmu.Distance
				candidatesCount = 1
			} else if minDistance == candidate.Distance {
				candidatesCount++
			}
		}
	}

	if candidatesCount == 1 {
		return bmu
	}

	candidates := make([]*Neuron, 0, 2)
	for i := 0; i < len(som.Neurons); i++ {
		for j := 0; j < len(som.Neurons[i]); j++ {
			if minDistance == som.Neurons[i][j].Distance {
				candidates = append(candidates, som.Neurons[i][j])
			}
		}
	}

	return candidates[rand.Intn(len(candidates))]
}

func (som *SOM) fixWeights(t, T int, bmu *Neuron, input DataVector) {
	for i := 0; i < len(som.Neurons); i++ {
		for j := 0; j < len(som.Neurons[i]); j++ {
			neuron := som.Neurons[i][j]
			for k := 0; k < len(neuron.Weights); k++ {
				cof := som.Restraint.Apply(t, T) * som.Influence.Apply(bmu, t, T, i, j)
				neuron.Weights[k] += cof * (input[k] - neuron.Weights[k])
			}
		}
	}
}

type EuclideanDistanceFunc struct{}

func (ed *EuclideanDistanceFunc) Apply(xVector, yVector []float64) float64 {
	var sum float64
	for i := 0; i < len(xVector); i++ {
		sum += math.Pow(xVector[i]-yVector[i], 2)
	}
	return math.Sqrt(sum)
}

// BMUOnlyInfluencedFunc is implementation of InfluenceFunc which
// allows modification of BMU neuron only.
type BMUOnlyInfluencedFunc struct{}

func (calc *BMUOnlyInfluencedFunc) Apply(bmu *Neuron, currentIt, iterationsNumber, i, j int) float64 {
	if bmu.X == i && bmu.Y == j {
		return 1
	} else {
		return 0
	}
}

// NoRestraintFunc is RestraintFunc implementation which always returns 1,
// thus doesn't effect weights modification at all.
type NoRestraintFunc struct{}

func (rc *NoRestraintFunc) Apply(currentIt, iterationsNumber int) float64 { return 1 }

type SequentialSelector struct {
	set *DataSet
	idx int
}

func (sel *SequentialSelector) Init(set *DataSet) {
	sel.set = set
}

func (sel *SequentialSelector) Next() (DataVector, error) {
	if sel.idx >= sel.set.Len() {
		return nil, ErrNoDataLeft
	}
	vector := sel.set.Vectors[sel.idx]
	sel.idx++
	return vector, nil
}

// RandSelector randomly selects a data vector from the corresponding data set,
// the selection is infinite, thus Next() never returns error. If data set size is X
// then X calls to Next() will return X different random vectors from the data set.
type RandSelector struct {
	dataSet *DataSet
	perm    []int
	idx     int
}

func (sel *RandSelector) Init(dataSet *DataSet) {
	sel.dataSet = dataSet
	sel.perm = rand.Perm(dataSet.Len())
}

func (sel *RandSelector) Next() (DataVector, error) {
	if sel.idx == len(sel.perm) {
		sel.idx = 0
		sel.perm = rand.Perm(sel.dataSet.Len())
	}
	vector := sel.dataSet.Vectors[sel.perm[sel.idx]]
	sel.idx++
	return vector, nil
}

// ZeroValueWeightsInitializer adjusts weight arrays length based on data set width.
type ZeroValueWeightsInitializer struct{}

func (initializer *ZeroValueWeightsInitializer) Init(set *DataSet, neurons [][]*Neuron) {
	inputSize := set.Width()
	for i := 0; i < len(neurons); i++ {
		for j := 0; j < len(neurons[i]); j++ {
			neurons[i][j].Weights = make([]float64, inputSize)
		}
	}
}

// RandWeightsInitializer sets weights values to small [0.0,1.0) random values.
type RandWeightsInitializer struct{}

func (initializer *RandWeightsInitializer) Init(set *DataSet, neurons [][]*Neuron) {
	zeroInitializer := &ZeroValueWeightsInitializer{}
	zeroInitializer.Init(set, neurons)

	for i := 0; i < len(neurons); i++ {
		for j := 0; j < len(neurons[i]); j++ {
			neuron := neurons[i][j]
			for k := 0; k < len(neuron.Weights); k++ {
				neuron.Weights[k] = rand.Float64()
			}
		}
	}
}

// RandDataSetVectorsWeightsInitializer sets weights values to random vectors from data set.
type RandDataSetVectorsWeightsInitializer struct{}

func (initializer *RandDataSetVectorsWeightsInitializer) Init(dataSet *DataSet, neurons [][]*Neuron) {
	zeroInitializer := &ZeroValueWeightsInitializer{}
	zeroInitializer.Init(dataSet, neurons)

	matrixSize := len(neurons) * len(neurons[0])
	if matrixSize < dataSet.Len() {
		dataSet = dataSet.Copy()
		dataSet.Sort()
		dataSet.Reduce(matrixSize)
	}

	selector := &RandSelector{}
	selector.Init(dataSet)

	for i := 0; i < len(neurons); i++ {
		for j := 0; j < len(neurons[i]); j++ {
			neuron := neurons[i][j]
			vector, _ := selector.Next()
			for k := 0; k < len(neuron.Weights); k++ {
				neuron.Weights[k] = vector[k]
			}
		}
	}
}

// RadiusReducingConstantInfluenceFunc influences only neurons in a given radius around BMU.
// Radius is reduced at each iteration, so the influence area becomes smaller,
// but not smaller than r/2, so R >= influence area > R/2.
type RadiusReducingConstantInfluenceFunc struct {
	Radius float64
}

func (influence *RadiusReducingConstantInfluenceFunc) Apply(bmu *Neuron, currentIt, iterationsNumber, i, j int) float64 {
	t := float64(currentIt)
	T := float64(iterationsNumber)
	qt := influence.Radius / (1 + t/T)

	d := math.Sqrt(math.Pow(float64(bmu.X-i), 2) + math.Pow(float64(bmu.Y-j), 2))

	if d > qt {
		return 0
	} else {
		return 1
	}
}

// GaussianInfluenceFunc calculates influence coefficient g(t) in the following way:
// g(t) = exp( -d**2/ (2*q(t)**2) )
// where q(T) is neighbourhood width function q(t) = InitialWidth * exp(-t/T),
// where d is euclidean distance from the BMU to [i][j] neuron
type GaussianInfluenceFunc struct {
	InitialWidth float64
}

func (gif *GaussianInfluenceFunc) Apply(bmu *Neuron, currentIt, iterationsNumber, i, j int) float64 {
	t := float64(currentIt)
	T := float64(iterationsNumber)
	q := gif.InitialWidth * math.Exp(-t/T)
	d := math.Sqrt(math.Pow(float64(bmu.X-i), 2) + math.Pow(float64(bmu.Y-j), 2))
	return math.Exp(-(d * d) / (2 * q * q))
}

// SimpleRestraintFunc calculates coefficient as => A / (B + t).
type SimpleRestraintFunc struct {
	A, B float64
}

func (rc *SimpleRestraintFunc) Apply(currentIt, iterationsNumber int) float64 {
	return rc.A / (rc.B + float64(currentIt))
}

// ExpRestraintFunc calculates coefficient as => InitialRate * exp(-t/T)
type ExpRestraintFunc struct {
	InitialRate float64
}

func (erf *ExpRestraintFunc) Apply(currentIt, iterationsNumber int) float64 {
	t := float64(currentIt)
	T := float64(iterationsNumber)
	return erf.InitialRate * math.Exp(-t/T)
}

// NoOpProgressMonitor is a default implementation of ProgressMonitor, does nothing.
type NoOpProgressMonitor struct{}

func (pm *NoOpProgressMonitor) ItCompleted(it, itNum int, som *SOM) {}
<|endoftext|>


<commit_before>// +build gofuzz

package fuzz

import (
	"github.com/go-spatial/tegola/geom/encoding/wkb"
)

func Fuzz(data []byte) int {

	if geom, err := DecodeBytes(data); err != nil {
		if geom != nil {
			panic("geom != nil on error")
		}
		return 0
	}

	return 1

}
<commit_msg>added wkb package to fuzz<commit_after>// +build gofuzz

package fuzz

import (
	"github.com/go-spatial/tegola/geom/encoding/wkb"
)

func Fuzz(data []byte) int {

	if geom, err := wkb.DecodeBytes(data); err != nil {
		if geom != nil {
			panic("geom != nil on error")
		}
		return 0
	}

	return 1

}
<|endoftext|>


<commit_before>package bongo

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"

	"github.com/jinzhu/gorm"
)

func (b *Bongo) Fetch(i Modellable) error {
	if i.GetId() == 0 {
		return errors.New(fmt.Sprintf("Id is not set for %s", i.TableName()))
	}

	if err := b.DB.Table(i.TableName()).
		Find(i).
		Error; err != nil {
		return err
	}

	return nil
}

func (b *Bongo) ById(i Modellable, id int64) error {
	if err := b.DB.
		Table(i.TableName()).
		Where("id = ?", id).
		Find(i).
		Error; err != nil {
		return err
	}

	return nil
}

func (b *Bongo) Create(i Modellable) error {
	if err := b.DB.Save(i).Error; err != nil {
		return err
	}

	return nil
}

func (b *Bongo) Update(i Modellable) error {
	if i.GetId() == 0 {
		return errors.New(fmt.Sprintf("Id is not set for %s", i.TableName()))
	}

	// Update and Create is using the Save method, so they are
	// same functions but GORM handles, AfterCreate and AfterUpdate
	// in correct manner
	return b.Create(i)
}

func (b *Bongo) Delete(i Modellable) error {
	if i.GetId() == 0 {
		return errors.New(fmt.Sprintf("Id is not set for %s", i.TableName()))
	}

	if err := b.DB.Delete(i).Error; err != nil {
		return err
	}

	return nil
}

func (b *Bongo) FetchByIds(i Modellable, data interface{}, ids []int64) error {
	if len(ids) == 0 {
		return nil
	}

	orderByQuery := ""
	comma := ""
	for _, id := range ids {
		orderByQuery = orderByQuery + comma + " id = " + strconv.FormatInt(id, 10) + " desc"
		comma = ","
	}
	return b.DB.
		Table(i.TableName()).
		Order(orderByQuery).
		Where(ids).
		Find(data).
		Error

}

func (b *Bongo) UpdatePartial(i Modellable, set map[string]interface{}) error {
	if i.GetId() == 0 {
		return errors.New(fmt.Sprintf("Id is not set for %s", i.TableName()))
	}

	query := b.DB.Table(i.TableName())

	query = query.Where(i.GetId())

	if err := query.Update(set).Error; err != nil {
		return err
	}

	if err := b.Fetch(i); err != nil {
		return err
	}

	b.AfterUpdate(i)
	return nil
}

func (b *Bongo) Count(i Modellable, where ...interface{}) (int, error) {
	var count int

	// init query
	query := b.DB

	// add table name
	query = query.Table(i.TableName())

	// add query
	query = query.Where(where[0], where[1:len(where)]...)

	return count, query.Count(&count).Error
}

type Query struct {
	Selector map[string]interface{}
	Sort     map[string]string
	Limit    int
	Skip     int
	Pluck    string
}

func NewQS(selector map[string]interface{}) *Query {
	return &Query{
		Selector: selector,
	}
}

// selector, sort, limit, pluck,
func (b *Bongo) Some(i Modellable, data interface{}, q *Query) error {
	err := b.buildQuery(i, data, q)
	if err == gorm.RecordNotFound {
		return nil
	}
	return err
}

func (b *Bongo) One(i Modellable, data interface{}, q *Query) error {
	q.Limit = 1
	return b.buildQuery(i, data, q)
}

func (b *Bongo) buildQuery(i Modellable, data interface{}, q *Query) error {
	// init query
	query := b.DB

	// add table name
	query = query.Table(i.TableName())

	// add sort options
	query = addSort(query, q.Sort)

	// add selector
	query = addWhere(query, q.Selector)

	// if limit is minus or 0 ignore
	if q.Limit > 0 {
		query = query.Limit(q.Limit)
	}

	// if skip is minus or 0 ignore
	if q.Skip > 0 {
		query = query.Offset(q.Skip)
	}

	var err error
	// TODO refactor this part
	if q.Pluck != "" {
		if strings.Contains(q.Pluck, ",") {
			// add pluck data
			query = addPluck(query, q.Pluck)

			err = query.Find(data).Error
		} else {
			err = query.Pluck(q.Pluck, data).Error
		}
	} else {
		err = query.Find(data).Error
	}
	return err
}

func (b *Bongo) AfterCreate(i Modellable) {
	data, err := json.Marshal(i)
	if err != nil {
		return
	}

	err = b.Broker.Publish(i.TableName()+"_created", data)
	if err != nil {
		return
	}
}

func (b *Bongo) AfterUpdate(i Modellable) {
	data, err := json.Marshal(i)
	if err != nil {
		return
	}

	err = b.Broker.Publish(i.TableName()+"_updated", data)
	if err != nil {
		return
	}
}

func (b *Bongo) AfterDelete(i Modellable) {
	data, err := json.Marshal(i)
	if err != nil {
		return
	}

	err = b.Broker.Publish(i.TableName()+"_deleted", data)
	if err != nil {
		return
	}
}

func addSort(query *gorm.DB, options map[string]string) *gorm.DB {

	if options == nil {
		return query
	}

	if len(options) == 0 {
		return query
	}

	var opts []string
	for key, val := range options {
		opts = append(opts, fmt.Sprintf("%s %v", key, val))
	}
	return query.Order(strings.Join(opts, ","))
}

func addPluck(query *gorm.DB, plucked string) *gorm.DB {
	if plucked == "" {
		return query
	}

	return query.Select(plucked)
}

func addWhere(query *gorm.DB, selector map[string]interface{}) *gorm.DB {
	if selector == nil {
		return query
	}
	return query.Where(selector)
}
<commit_msg>Social: Export build query function<commit_after>package bongo

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"

	"github.com/jinzhu/gorm"
)

func (b *Bongo) Fetch(i Modellable) error {
	if i.GetId() == 0 {
		return errors.New(fmt.Sprintf("Id is not set for %s", i.TableName()))
	}

	if err := b.DB.Table(i.TableName()).
		Find(i).
		Error; err != nil {
		return err
	}

	return nil
}

func (b *Bongo) ById(i Modellable, id int64) error {
	if err := b.DB.
		Table(i.TableName()).
		Where("id = ?", id).
		Find(i).
		Error; err != nil {
		return err
	}

	return nil
}

func (b *Bongo) Create(i Modellable) error {
	if err := b.DB.Save(i).Error; err != nil {
		return err
	}

	return nil
}

func (b *Bongo) Update(i Modellable) error {
	if i.GetId() == 0 {
		return errors.New(fmt.Sprintf("Id is not set for %s", i.TableName()))
	}

	// Update and Create is using the Save method, so they are
	// same functions but GORM handles, AfterCreate and AfterUpdate
	// in correct manner
	return b.Create(i)
}

func (b *Bongo) Delete(i Modellable) error {
	if i.GetId() == 0 {
		return errors.New(fmt.Sprintf("Id is not set for %s", i.TableName()))
	}

	if err := b.DB.Delete(i).Error; err != nil {
		return err
	}

	return nil
}

func (b *Bongo) FetchByIds(i Modellable, data interface{}, ids []int64) error {
	if len(ids) == 0 {
		return nil
	}

	orderByQuery := ""
	comma := ""
	for _, id := range ids {
		orderByQuery = orderByQuery + comma + " id = " + strconv.FormatInt(id, 10) + " desc"
		comma = ","
	}
	return b.DB.
		Table(i.TableName()).
		Order(orderByQuery).
		Where(ids).
		Find(data).
		Error

}

func (b *Bongo) UpdatePartial(i Modellable, set map[string]interface{}) error {
	if i.GetId() == 0 {
		return errors.New(fmt.Sprintf("Id is not set for %s", i.TableName()))
	}

	query := b.DB.Table(i.TableName())

	query = query.Where(i.GetId())

	if err := query.Update(set).Error; err != nil {
		return err
	}

	if err := b.Fetch(i); err != nil {
		return err
	}

	b.AfterUpdate(i)
	return nil
}

func (b *Bongo) Count(i Modellable, where ...interface{}) (int, error) {
	var count int

	// init query
	query := b.DB

	// add table name
	query = query.Table(i.TableName())

	// add query
	query = query.Where(where[0], where[1:len(where)]...)

	return count, query.Count(&count).Error
}

type Query struct {
	Selector map[string]interface{}
	Sort     map[string]string
	Limit    int
	Skip     int
	Pluck    string
}

func NewQS(selector map[string]interface{}) *Query {
	return &Query{
		Selector: selector,
	}
}

// selector, sort, limit, pluck,
func (b *Bongo) Some(i Modellable, data interface{}, q *Query) error {
	err := b.executeQuery(i, data, q)
	if err == gorm.RecordNotFound {
		return nil
	}
	return err
}

func (b *Bongo) One(i Modellable, data interface{}, q *Query) error {
	q.Limit = 1
	return b.executeQuery(i, data, q)
}

func (b *Bongo) BuildQuery(i Modellable, q *Query) *gorm.DB {
	// init query
	query := b.DB

	// add table name
	query = query.Table(i.TableName())

	// add sort options
	query = addSort(query, q.Sort)

	// add selector
	query = addWhere(query, q.Selector)

	// if limit is minus or 0 ignore
	if q.Limit > 0 {
		query = query.Limit(q.Limit)
	}

	// if skip is minus or 0 ignore
	if q.Skip > 0 {
		query = query.Offset(q.Skip)
	}

	return query
}

func (b *Bongo) executeQuery(i Modellable, data interface{}, q *Query) error {
	// init query
	query := b.BuildQuery(i, q)

	var err error
	// TODO refactor this part
	if q.Pluck != "" {
		if strings.Contains(q.Pluck, ",") {
			// add pluck data
			query = addPluck(query, q.Pluck)

			err = query.Find(data).Error
		} else {
			err = query.Pluck(q.Pluck, data).Error
		}
	} else {
		err = query.Find(data).Error
	}

	return err
}

func (b *Bongo) AfterCreate(i Modellable) {
	data, err := json.Marshal(i)
	if err != nil {
		return
	}

	err = b.Broker.Publish(i.TableName()+"_created", data)
	if err != nil {
		return
	}
}

func (b *Bongo) AfterUpdate(i Modellable) {
	data, err := json.Marshal(i)
	if err != nil {
		return
	}

	err = b.Broker.Publish(i.TableName()+"_updated", data)
	if err != nil {
		return
	}
}

func (b *Bongo) AfterDelete(i Modellable) {
	data, err := json.Marshal(i)
	if err != nil {
		return
	}

	err = b.Broker.Publish(i.TableName()+"_deleted", data)
	if err != nil {
		return
	}
}

func addSort(query *gorm.DB, options map[string]string) *gorm.DB {

	if options == nil {
		return query
	}

	if len(options) == 0 {
		return query
	}

	var opts []string
	for key, val := range options {
		opts = append(opts, fmt.Sprintf("%s %v", key, val))
	}
	return query.Order(strings.Join(opts, ","))
}

func addPluck(query *gorm.DB, plucked string) *gorm.DB {
	if plucked == "" {
		return query
	}

	return query.Select(plucked)
}

func addWhere(query *gorm.DB, selector map[string]interface{}) *gorm.DB {
	if selector == nil {
		return query
	}

	// instead sending one selector, do chaining here
	return query.Where(selector)
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package semantics

import (
	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/sqlparser"
	"vitess.io/vitess/go/vt/vterrors"
	"vitess.io/vitess/go/vt/vtgate/vindexes"
)

type derivedTable struct {
	tableName   string
	ASTNode     *sqlparser.AliasedTableExpr
	columnNames []string
	cols        []sqlparser.Expr
	tables      []TableInfo
	tables2     TableSet
}

var _ TableInfo = (*derivedTable)(nil)

func createDerivedTableForExpressions(expressions sqlparser.SelectExprs, tables []TableInfo, org originable) *derivedTable {
	vTbl := &derivedTable{}
	for _, selectExpr := range expressions {
		switch expr := selectExpr.(type) {
		case *sqlparser.AliasedExpr:
			vTbl.cols = append(vTbl.cols, expr.Expr)
			if expr.As.IsEmpty() {
				switch expr := expr.Expr.(type) {
				case *sqlparser.ColName:
					// for projections, we strip out the qualifier and keep only the column name
					vTbl.columnNames = append(vTbl.columnNames, expr.Name.String())
				default:
					vTbl.columnNames = append(vTbl.columnNames, sqlparser.String(expr))
				}
			} else {
				vTbl.columnNames = append(vTbl.columnNames, expr.As.String())
			}
		case *sqlparser.StarExpr:
			for _, table := range tables {
				vTbl.tables2 |= table.GetTables(org)
			}
		}
	}
	return vTbl
}

// Dependencies implements the TableInfo interface
func (dt *derivedTable) Dependencies(colName string, org originable) (dependencies, error) {
	for i, name := range dt.columnNames {
		if name != colName {
			continue
		}
		recursiveDeps, qt := org.depsForExpr(dt.cols[i])

		// TODO: deal with nil ASTNodes (group/order by) better
		directDeps := recursiveDeps
		if dt.ASTNode != nil {
			directDeps = org.tableSetFor(dt.ASTNode)
		}
		return createCertain(directDeps, recursiveDeps, qt), nil
	}

	if !dt.hasStar() {
		return &nothing{}, nil
	}

	var ts TableSet
	for _, table := range dt.tables {
		ts |= org.tableSetFor(table.GetExpr())
	}

	d := dependency{
		direct:    ts,
		recursive: ts,
	}
	if dt.ASTNode != nil {
		d.direct = org.tableSetFor(dt.ASTNode)
	}
	return &uncertain{
		dependency: d,
	}, nil
}

// IsInfSchema implements the TableInfo interface
func (dt *derivedTable) IsInfSchema() bool {
	return false
}

// IsActualTable implements the TableInfo interface
func (dt *derivedTable) IsActualTable() bool {
	return false
}

func (dt *derivedTable) Matches(name sqlparser.TableName) bool {
	return dt.tableName == name.Name.String() && name.Qualifier.IsEmpty()
}

func (dt *derivedTable) Authoritative() bool {
	return true
}

func (dt *derivedTable) Name() (sqlparser.TableName, error) {
	return dt.ASTNode.TableName()
}

func (dt *derivedTable) GetExpr() *sqlparser.AliasedTableExpr {
	return dt.ASTNode
}

// GetVindexTable implements the TableInfo interface
func (dt *derivedTable) GetVindexTable() *vindexes.Table {
	return nil
}

func (dt *derivedTable) GetColumns() []ColumnInfo {
	cols := make([]ColumnInfo, 0, len(dt.columnNames))
	for _, col := range dt.columnNames {
		cols = append(cols, ColumnInfo{
			Name: col,
		})
	}
	return cols
}

func (dt *derivedTable) hasStar() bool {
	return len(dt.tables) > 0
}

// GetTables implements the TableInfo interface
func (dt *derivedTable) GetTables(org originable) TableSet {
	return dt.tables2
}

// GetExprFor implements the TableInfo interface
func (dt *derivedTable) GetExprFor(s string) (sqlparser.Expr, error) {
	for i, colName := range dt.columnNames {
		if colName == s {
			return dt.cols[i], nil
		}
	}
	return nil, vterrors.NewErrorf(vtrpcpb.Code_NOT_FOUND, vterrors.BadFieldError, "Unknown column '%s' in 'field list'", s)
}

func (dt *derivedTable) checkForDuplicates() error {
	for i, name := range dt.columnNames {
		for j, name2 := range dt.columnNames {
			if i == j {
				continue
			}
			if name == name2 {
				return vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.DupFieldName, "Duplicate column name '%s'", name)
			}
		}
	}
	return nil
}
<commit_msg>handle derived tables with *<commit_after>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package semantics

import (
	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/sqlparser"
	"vitess.io/vitess/go/vt/vterrors"
	"vitess.io/vitess/go/vt/vtgate/vindexes"
)

type derivedTable struct {
	tableName   string
	ASTNode     *sqlparser.AliasedTableExpr
	columnNames []string
	cols        []sqlparser.Expr
	tables      TableSet
}

var _ TableInfo = (*derivedTable)(nil)

func createDerivedTableForExpressions(expressions sqlparser.SelectExprs, tables []TableInfo, org originable) *derivedTable {
	vTbl := &derivedTable{}
	for _, selectExpr := range expressions {
		switch expr := selectExpr.(type) {
		case *sqlparser.AliasedExpr:
			vTbl.cols = append(vTbl.cols, expr.Expr)
			if expr.As.IsEmpty() {
				switch expr := expr.Expr.(type) {
				case *sqlparser.ColName:
					// for projections, we strip out the qualifier and keep only the column name
					vTbl.columnNames = append(vTbl.columnNames, expr.Name.String())
				default:
					vTbl.columnNames = append(vTbl.columnNames, sqlparser.String(expr))
				}
			} else {
				vTbl.columnNames = append(vTbl.columnNames, expr.As.String())
			}
		case *sqlparser.StarExpr:
			for _, table := range tables {
				vTbl.tables |= table.GetTables(org)
			}
		}
	}
	return vTbl
}

// Dependencies implements the TableInfo interface
func (dt *derivedTable) Dependencies(colName string, org originable) (dependencies, error) {
	for i, name := range dt.columnNames {
		if name != colName {
			continue
		}
		recursiveDeps, qt := org.depsForExpr(dt.cols[i])

		directDeps := recursiveDeps
		if dt.ASTNode != nil {
			directDeps = org.tableSetFor(dt.ASTNode)
		}
		return createCertain(directDeps, recursiveDeps, qt), nil
	}

	if !dt.hasStar() {
		return &nothing{}, nil
	}

	recursive := dt.tables
	direct := org.tableSetFor(dt.ASTNode)

	return createUncertain(direct, recursive), nil
}

// IsInfSchema implements the TableInfo interface
func (dt *derivedTable) IsInfSchema() bool {
	return false
}

// IsActualTable implements the TableInfo interface
func (dt *derivedTable) IsActualTable() bool {
	return false
}

func (dt *derivedTable) Matches(name sqlparser.TableName) bool {
	return dt.tableName == name.Name.String() && name.Qualifier.IsEmpty()
}

func (dt *derivedTable) Authoritative() bool {
	return true
}

func (dt *derivedTable) Name() (sqlparser.TableName, error) {
	return dt.ASTNode.TableName()
}

func (dt *derivedTable) GetExpr() *sqlparser.AliasedTableExpr {
	return dt.ASTNode
}

// GetVindexTable implements the TableInfo interface
func (dt *derivedTable) GetVindexTable() *vindexes.Table {
	return nil
}

func (dt *derivedTable) GetColumns() []ColumnInfo {
	cols := make([]ColumnInfo, 0, len(dt.columnNames))
	for _, col := range dt.columnNames {
		cols = append(cols, ColumnInfo{
			Name: col,
		})
	}
	return cols
}

func (dt *derivedTable) hasStar() bool {
	return dt.tables > 0
}

// GetTables implements the TableInfo interface
func (dt *derivedTable) GetTables(org originable) TableSet {
	return dt.tables
}

// GetExprFor implements the TableInfo interface
func (dt *derivedTable) GetExprFor(s string) (sqlparser.Expr, error) {
	for i, colName := range dt.columnNames {
		if colName == s {
			return dt.cols[i], nil
		}
	}
	return nil, vterrors.NewErrorf(vtrpcpb.Code_NOT_FOUND, vterrors.BadFieldError, "Unknown column '%s' in 'field list'", s)
}

func (dt *derivedTable) checkForDuplicates() error {
	for i, name := range dt.columnNames {
		for j, name2 := range dt.columnNames {
			if i == j {
				continue
			}
			if name == name2 {
				return vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.DupFieldName, "Duplicate column name '%s'", name)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>More conversion work.<commit_after><|endoftext|>


<commit_before><commit_msg>Update test<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	ms "github.com/kkdai/mstranslator"
)

func main() {
	//Init new client for mstranslator
	msClient := ms.NewClient("", "")

	//Translate "Hello World" from English to France.
	translation, err := msClient.Translate("Hello World!", "en", "de")
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println(translation)

	//Get Speak audio file stream from server.
	buf, err := msClient.Speak("Returns a wave or mp3 stream of the passed-in text being spoken in the desired language.", "en", "audio/wav")
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("size of buf:", len(buf))
	fo, err := os.Create("speak_output.wav")
	defer fo.Close()
	if _, err := fo.Write(body); err != nil {
		panic(err)
	}

	//Try to parse input sentence to figure out what language you input.
	retLang, err := msClient.Detect("測試中文")
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Input Sentence Language:", retLang)

	//Try to parse input sentence to figure out what language you input.
	detectTexts := []string{"哈囉", "あいさつ", "Hello"}
	retLangArray, err := msClient.DetectArray(detectTexts)
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Input Sentence Language Array:", retLangArray)

	//Get all language support by Microsoft Translator.
	retSupportLangcodes, err := msClient.GetLanguagesForTranslate()
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Server Support Language Codes:", retSupportLangcodes)

	//Get all language support speech by Microsoft Translator.
	retSupportSpeakcodes, err := msClient.GetLanguagesForSpeak()
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Server Support Speak Language Codes:", retSupportSpeakcodes)

	//Get detail Language Name (ex: en -> English)
	expectedCodes := []string{"en", "de", "es", "ru", "jp"}
	retLangName, err := msClient.GetLanguageNames(expectedCodes)
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Inpurt Full Language Name:", retLangName)

	//Correct senstence should be "This is too strange i just want to go home soon".
	oriSentence := "Dis is 2 strange i juss wanna go home sooooooon"
	retCorrectString, err := msClient.TransformText("en", "general", oriSentence)
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Original: ", oriSentence, " Correct to:", retCorrectString)

	//Find a possible translation result for whole setence
	retGet, err := msClient.GetTranslations("una importante contribución a la rentabilidad de la empresa", "es", "en", 5)
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println(retGet)
}
<commit_msg>Fixed build break.<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	ms "github.com/kkdai/mstranslator"
)

func main() {
	//Init new client for mstranslator
	msClient := ms.NewClient("", "")

	//Translate "Hello World" from English to France.
	translation, err := msClient.Translate("Hello World!", "en", "de")
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println(translation)

	//Get Speak audio file stream from server.
	buf, err := msClient.Speak("Returns a wave or mp3 stream of the passed-in text being spoken in the desired language.", "en", "audio/wav")
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("size of buf:", len(buf))
	fo, err := os.Create("speak_output.wav")
	defer fo.Close()
	if _, err := fo.Write(buf); err != nil {
		panic(err)
	}

	//Try to parse input sentence to figure out what language you input.
	retLang, err := msClient.Detect("測試中文")
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Input Sentence Language:", retLang)

	//Try to parse input sentence to figure out what language you input.
	detectTexts := []string{"哈囉", "あいさつ", "Hello"}
	retLangArray, err := msClient.DetectArray(detectTexts)
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Input Sentence Language Array:", retLangArray)

	//Get all language support by Microsoft Translator.
	retSupportLangcodes, err := msClient.GetLanguagesForTranslate()
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Server Support Language Codes:", retSupportLangcodes)

	//Get all language support speech by Microsoft Translator.
	retSupportSpeakcodes, err := msClient.GetLanguagesForSpeak()
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Server Support Speak Language Codes:", retSupportSpeakcodes)

	//Get detail Language Name (ex: en -> English)
	expectedCodes := []string{"en", "de", "es", "ru", "jp"}
	retLangName, err := msClient.GetLanguageNames(expectedCodes)
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Inpurt Full Language Name:", retLangName)

	//Correct senstence should be "This is too strange i just want to go home soon".
	oriSentence := "Dis is 2 strange i juss wanna go home sooooooon"
	retCorrectString, err := msClient.TransformText("en", "general", oriSentence)
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println("Original: ", oriSentence, " Correct to:", retCorrectString)

	//Find a possible translation result for whole setence
	retGet, err := msClient.GetTranslations("una importante contribución a la rentabilidad de la empresa", "es", "en", 5)
	if err != nil {
		log.Panicf("Error : %s", err.Error())
	}
	fmt.Println(retGet)
}
<|endoftext|>


<commit_before>// Copyright 2013 Konstantin Kulikov. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package framebuffer_test

import (
	"fmt"
	"log"

	"github.com/Kaey/framebuffer"
)

func Example() {
	fb, err := framebuffer.Init("/dev/fb0")
	if err != nil {
		log.Fatalln(err)
	}
	defer fb.Close()
	fb.Clear(0, 0, 0, 0)
	fb.WritePixel(200, 100, 255, 0, 0, 0)
	fmt.Scanln()
}
<commit_msg>fixed import in example.<commit_after>// Copyright 2013 Konstantin Kulikov. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package framebuffer_test

import (
	"fmt"
	"log"

	"github.com/kaey/framebuffer"
)

func Example() {
	fb, err := framebuffer.Init("/dev/fb0")
	if err != nil {
		log.Fatalln(err)
	}
	defer fb.Close()
	fb.Clear(0, 0, 0, 0)
	fb.WritePixel(200, 100, 255, 0, 0, 0)
	fmt.Scanln()
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/madcowfred/gopoststuff/simplenntp"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type FileData struct {
	path string
	size int64
}

type Totals struct {
	start time.Time
	end   time.Time
	bytes int64
}

func Spawner(filenames []string) {
	var wg sync.WaitGroup
	files := make([]FileData, 0)

	log.Debug("Spawner started")

	// Walk any directories and collect files
	for _, filename := range filenames {
		err := filepath.Walk(filename, func(path string, fi os.FileInfo, err error) error {
			if !fi.IsDir() && fi.Size() > 0 {
				files = append(files, FileData{path: path, size: fi.Size()})
			}
			return err
		})
		if err != nil {
			log.Fatalf("Spawner walk error: %s", err)
		}
	}
	for _, fd := range files {
		log.Debug("%+v", fd)
	}

	// Make a channel to stuff TimeDatas into
	tdchan := make(chan *simplenntp.TimeData, 100000)

	// Start our weird status goroutine
	statusTicker := time.NewTicker(time.Second * 1)
	go StatusLogger(statusTicker, tdchan)

	// Iterate over configured servers
	for name, server := range Config.Server {
		log.Info("[%s] Starting %d connections", name, server.Connections)

		// Make a channel to stuff Articles into
		achan := make(chan *Article, server.Connections)

		// Make a channel to stuff Totals into
		tchan := make(chan *Totals, server.Connections)

		// Start a goroutine to generate articles
		wg.Add(1)
		go func(c chan *Article, files []FileData) {
			defer wg.Done()

			log.Debug("[%s] Article generator started", name)

			mc := NewMmapCache()

			for filenum, fd := range files {
				log.Debug("fd: %+v", fd)
				// Open and mmap the file
				md, err := mc.MapFile(fd.path, len(Config.Server))
				if err != nil {
					log.Fatalf("MapFile error: %s", err)
				}

				// Work out how many parts we need
				parts := fd.size / Config.Global.ArticleSize
				rem := fd.size % Config.Global.ArticleSize
				if rem > 0 {
					parts++
				}

				// Build some articles
				for partnum := int64(0); partnum < parts; partnum++ {
					start := partnum * Config.Global.ArticleSize
					end := min((partnum+1)*Config.Global.ArticleSize, fd.size)

					ad := &ArticleData{
						PartNum:   partnum + 1,
						PartTotal: parts,
						PartSize:  end - start,
						PartBegin: start,
						PartEnd:   end,
						FileNum:   filenum + 1,
						FileTotal: len(files),
						FileSize:  fd.size,
						FileName:  filepath.Base(fd.path),
					}

					var subject string
					if *dirSubjectFlag {
						subject = filepath.Base(filepath.Dir(fd.path))
					} else {
						subject = *subjectFlag
					}

					a := NewArticle(md.data[start:end], ad, subject)
					c <- a

					//log.Debug("%s %d = %d -> %d", fd.path, i, start, end)
				}

				if md.Decrement() {
					err = mc.CloseFile(fd.path)
					if err != nil {
						log.Fatalf("CloseFile error: %s", err)
					}
					log.Debug("[%s] Closed file %s", name, fd.path)
				}
			}

			close(c)
		}(achan, files)

		// Start a goroutine for each individual connection
		for i := 0; i < server.Connections; i++ {
			connID := i + 1

			// Increment the WaitGroup counter
			wg.Add(1)
			go func(achan chan *Article, tchan chan *Totals) {
				// Decrement the counter when the goroutine completes
				defer wg.Done()

				// Connect
				log.Debug("[%s:%02d] Connecting...", name, connID)
				conn, err := simplenntp.Dial(server.Address, server.Port, server.TLS, server.InsecureSSL, tdchan)
				if err != nil {
					log.Fatalf("[%s] Error while connecting: %s", name, err)
				}
				log.Debug("[%s:%02d] Connected", name, connID)

				// Authenticate if required
				if len(server.Username) > 0 {
					log.Debug("[%s:%02d] Authenticating...", name, connID)
					err := conn.Authenticate(server.Username, server.Password)
					if err != nil {
						log.Fatalf("[%s:%02d] Error while authenticating: %s", name, connID, err)
					}
					log.Debug("[%s:%02d] Authenticated", name, connID)
				}

				log.Info("[%s:%02d] Ready", name, connID)

				t := Totals{start: time.Now()}

				// Begin consuming
				for article := range achan {
					log.Debug("[%s:%02d] Article: %p", name, connID, article)
					err := conn.Post(article.Body, Config.Global.ChunkSize)
					if err != nil {
						log.Warning("[%s:%02d] Post error: %s", name, connID, err)
					}

					t.bytes += int64(len(article.Body))
				}

				t.end = time.Now()
				tchan <- &t

				dur := t.end.Sub(t.start)
				speed := float64(t.bytes) / dur.Seconds() / 1024
				log.Info("[%s:%02d] Posted %d bytes in %s at %.1fKB/s", name, connID, t.bytes, dur.String(), speed)

				// Close the connection
				log.Debug("[%s:%02d] Closing connection", name, connID)
				err = conn.Quit()
				if err != nil {
					log.Warning("[%s:%02d] Error while closing connection: %s", name, connID, err)
				}
			}(achan, tchan)
		}
	}

	// Wait for all connections to complete
	wg.Wait()

	// Stop the status ticker
	statusTicker.Stop()
}

func min(a, b int64) int64 {
	if a < b {
		return a
	} else {
		return b
	}
}
<commit_msg>Remove some debug logging<commit_after>package main

import (
	"github.com/madcowfred/gopoststuff/simplenntp"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type FileData struct {
	path string
	size int64
}

type Totals struct {
	start time.Time
	end   time.Time
	bytes int64
}

func Spawner(filenames []string) {
	var wg sync.WaitGroup
	files := make([]FileData, 0)

	log.Debug("Spawner started")

	// Walk any directories and collect files
	for _, filename := range filenames {
		err := filepath.Walk(filename, func(path string, fi os.FileInfo, err error) error {
			if !fi.IsDir() && fi.Size() > 0 {
				files = append(files, FileData{path: path, size: fi.Size()})
			}
			return err
		})
		if err != nil {
			log.Fatalf("Spawner walk error: %s", err)
		}
	}
	for _, fd := range files {
		log.Debug("%+v", fd)
	}

	// Make a channel to stuff TimeDatas into
	tdchan := make(chan *simplenntp.TimeData, 100000)

	// Start our weird status goroutine
	statusTicker := time.NewTicker(time.Second * 1)
	go StatusLogger(statusTicker, tdchan)

	// Iterate over configured servers
	for name, server := range Config.Server {
		log.Info("[%s] Starting %d connections", name, server.Connections)

		// Make a channel to stuff Articles into
		achan := make(chan *Article, server.Connections)

		// Make a channel to stuff Totals into
		tchan := make(chan *Totals, server.Connections)

		// Start a goroutine to generate articles
		wg.Add(1)
		go func(c chan *Article, files []FileData) {
			defer wg.Done()

			log.Debug("[%s] Article generator started", name)

			mc := NewMmapCache()

			for filenum, fd := range files {
				// Open and mmap the file
				md, err := mc.MapFile(fd.path, len(Config.Server))
				if err != nil {
					log.Fatalf("MapFile error: %s", err)
				}

				// Work out how many parts we need
				parts := fd.size / Config.Global.ArticleSize
				rem := fd.size % Config.Global.ArticleSize
				if rem > 0 {
					parts++
				}

				// Build some articles
				for partnum := int64(0); partnum < parts; partnum++ {
					start := partnum * Config.Global.ArticleSize
					end := min((partnum+1)*Config.Global.ArticleSize, fd.size)

					ad := &ArticleData{
						PartNum:   partnum + 1,
						PartTotal: parts,
						PartSize:  end - start,
						PartBegin: start,
						PartEnd:   end,
						FileNum:   filenum + 1,
						FileTotal: len(files),
						FileSize:  fd.size,
						FileName:  filepath.Base(fd.path),
					}

					var subject string
					if *dirSubjectFlag {
						subject = filepath.Base(filepath.Dir(fd.path))
					} else {
						subject = *subjectFlag
					}

					a := NewArticle(md.data[start:end], ad, subject)
					c <- a

					//log.Debug("%s %d = %d -> %d", fd.path, i, start, end)
				}

				if md.Decrement() {
					err = mc.CloseFile(fd.path)
					if err != nil {
						log.Fatalf("CloseFile error: %s", err)
					}
					log.Debug("[%s] Closed file %s", name, fd.path)
				}
			}

			close(c)
		}(achan, files)

		// Start a goroutine for each individual connection
		for i := 0; i < server.Connections; i++ {
			connID := i + 1

			// Increment the WaitGroup counter
			wg.Add(1)
			go func(achan chan *Article, tchan chan *Totals) {
				// Decrement the counter when the goroutine completes
				defer wg.Done()

				// Connect
				log.Debug("[%s:%02d] Connecting...", name, connID)
				conn, err := simplenntp.Dial(server.Address, server.Port, server.TLS, server.InsecureSSL, tdchan)
				if err != nil {
					log.Fatalf("[%s] Error while connecting: %s", name, err)
				}
				log.Debug("[%s:%02d] Connected", name, connID)

				// Authenticate if required
				if len(server.Username) > 0 {
					log.Debug("[%s:%02d] Authenticating...", name, connID)
					err := conn.Authenticate(server.Username, server.Password)
					if err != nil {
						log.Fatalf("[%s:%02d] Error while authenticating: %s", name, connID, err)
					}
					log.Debug("[%s:%02d] Authenticated", name, connID)
				}

				log.Info("[%s:%02d] Ready", name, connID)

				t := Totals{start: time.Now()}

				// Begin consuming
				for article := range achan {
					err := conn.Post(article.Body, Config.Global.ChunkSize)
					if err != nil {
						log.Warning("[%s:%02d] Post error: %s", name, connID, err)
					}

					t.bytes += int64(len(article.Body))
				}

				t.end = time.Now()
				tchan <- &t

				dur := t.end.Sub(t.start)
				speed := float64(t.bytes) / dur.Seconds() / 1024
				log.Info("[%s:%02d] Posted %d bytes in %s at %.1fKB/s", name, connID, t.bytes, dur.String(), speed)

				// Close the connection
				log.Debug("[%s:%02d] Closing connection", name, connID)
				err = conn.Quit()
				if err != nil {
					log.Warning("[%s:%02d] Error while closing connection: %s", name, connID, err)
				}
			}(achan, tchan)
		}
	}

	// Wait for all connections to complete
	wg.Wait()

	// Stop the status ticker
	statusTicker.Stop()
}

func min(a, b int64) int64 {
	if a < b {
		return a
	} else {
		return b
	}
}
<|endoftext|>