<commit_before>package sakuracloud

import (
	"errors"
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccSakuraCloudDataSourceArchive_Basic(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:                  func() { testAccPreCheck(t) },
		Providers:                 testAccProviders,
		PreventPostDestroyRefresh: true,
		CheckDestroy:              testAccCheckSakuraCloudArchiveDataSourceDestroy,

		Steps: []resource.TestStep{
			{
				Config: testAccCheckSakuraCloudDataSourceArchiveConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceID("data.sakuracloud_archive.foobar"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "name", "Ubuntu Server 16.04.4 LTS 64bit"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "size", "20"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "zone", "tk1v"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.#", "5"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.0", "@size-extendable"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.1", "arch-64bit"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.2", "distro-ubuntu"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.3", "distro-ver-16.04.4"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.4", "os-linux"),
				),
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchive_OSType,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceID("data.sakuracloud_archive.foobar"),
				),
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchiveConfig_With_Tag,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceID("data.sakuracloud_archive.foobar"),
				),
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchive_NameSelector_Exists,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceID("data.sakuracloud_archive.foobar"),
				),
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchive_TagSelector_Exists,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceID("data.sakuracloud_archive.foobar"),
				),
			},

			{
				Config: testAccCheckSakuraCloudDataSourceArchiveConfig_NotExists,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceNotExists("data.sakuracloud_archive.foobar"),
				),
				Destroy: true,
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchiveConfig_With_NotExists_Tag,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceNotExists("data.sakuracloud_archive.foobar"),
				),
				Destroy: true,
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchive_NameSelector_NotExists,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceNotExists("data.sakuracloud_archive.foobar"),
				),
				Destroy: true,
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchive_TagSelector_NotExists,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceNotExists("data.sakuracloud_archive.foobar"),
				),
				Destroy: true,
			},
		},
	})
}

func testAccCheckSakuraCloudArchiveDataSourceID(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Can't find Archive data source: %s", n)
		}

		if rs.Primary.ID == "" {
			return errors.New("Archive data source ID not set")
		}
		return nil
	}
}

func testAccCheckSakuraCloudArchiveDataSourceNotExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		_, ok := s.RootModule().Resources[n]
		if ok {
			return fmt.Errorf("Found Archive data source: %s", n)
		}
		return nil
	}
}

func testAccCheckSakuraCloudArchiveDataSourceDestroy(s *terraform.State) error {
	client := testAccProvider.Meta().(*APIClient)
	originalZone := client.Zone
	client.Zone = "tk1v"
	defer func() { client.Zone = originalZone }()

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "sakuracloud_archive" {
			continue
		}

		if rs.Primary.ID == "" {
			continue
		}

		_, err := client.Archive.Read(toSakuraCloudID(rs.Primary.ID))

		if err == nil {
			return errors.New("Archive still exists")
		}
	}

	return nil
}

var testAccCheckSakuraCloudDataSourceArchiveConfig = `
data "sakuracloud_archive" "foobar" {
    filter = {
	name = "Name"
	values = ["Ubuntu Server 16"]
    }
    zone = "tk1v"
}`

var testAccCheckSakuraCloudDataSourceArchiveConfig_With_Tag = `
data "sakuracloud_archive" "foobar" {
    filter = {
	name = "Tags"
	values = ["distro-ubuntu","os-linux"]
    }
    zone = "tk1v"
}`

var testAccCheckSakuraCloudDataSourceArchiveConfig_With_NotExists_Tag = `
data "sakuracloud_archive" "foobar" {
    filter = {
	name = "Tags"
	values = ["distro-ubuntu-xxxxxxxxxxx","os-linux-xxxxxxxx"]
    }
    zone = "tk1v"
}`

var testAccCheckSakuraCloudDataSourceArchiveConfig_NotExists = `
data "sakuracloud_archive" "foobar" {
    filter = {
	name = "Name"
	values = ["xxxxxxxxxxxxxxxxxx"]
    }
    zone = "tk1v"
}`

var testAccCheckSakuraCloudDataSourceArchive_OSType = `
data "sakuracloud_archive" "foobar" {
    os_type = "rancheros"
    zone = "tk1v"
}
`

var testAccCheckSakuraCloudDataSourceArchive_NameSelector_Exists = `
data "sakuracloud_archive" "foobar" {
    name_selectors = ["Ubuntu", "Server","16"]
    zone = "tk1v"
}
`
var testAccCheckSakuraCloudDataSourceArchive_NameSelector_NotExists = `
data "sakuracloud_archive" "foobar" {
    name_selectors = ["xxxxxxxxxx"]
    zone = "tk1v"
}
`

var testAccCheckSakuraCloudDataSourceArchive_TagSelector_Exists = `
data "sakuracloud_archive" "foobar" {
	tag_selectors = ["distro-ubuntu","os-linux"]
    zone = "tk1v"
}`

var testAccCheckSakuraCloudDataSourceArchive_TagSelector_NotExists = `
data "sakuracloud_archive" "foobar" {
	tag_selectors = ["xxxxxxxxxx"]
    zone = "tk1v"
}`
<commit_msg>Update test values from ubuntu 16.04.4 to 16.04.5<commit_after>package sakuracloud

import (
	"errors"
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccSakuraCloudDataSourceArchive_Basic(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:                  func() { testAccPreCheck(t) },
		Providers:                 testAccProviders,
		PreventPostDestroyRefresh: true,
		CheckDestroy:              testAccCheckSakuraCloudArchiveDataSourceDestroy,

		Steps: []resource.TestStep{
			{
				Config: testAccCheckSakuraCloudDataSourceArchiveConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceID("data.sakuracloud_archive.foobar"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "name", "Ubuntu Server 16.04.5 LTS 64bit"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "size", "20"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "zone", "tk1v"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.#", "5"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.0", "@size-extendable"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.1", "arch-64bit"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.2", "distro-ubuntu"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.3", "distro-ver-16.04.4"),
					resource.TestCheckResourceAttr("data.sakuracloud_archive.foobar", "tags.4", "os-linux"),
				),
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchive_OSType,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceID("data.sakuracloud_archive.foobar"),
				),
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchiveConfig_With_Tag,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceID("data.sakuracloud_archive.foobar"),
				),
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchive_NameSelector_Exists,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceID("data.sakuracloud_archive.foobar"),
				),
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchive_TagSelector_Exists,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceID("data.sakuracloud_archive.foobar"),
				),
			},

			{
				Config: testAccCheckSakuraCloudDataSourceArchiveConfig_NotExists,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceNotExists("data.sakuracloud_archive.foobar"),
				),
				Destroy: true,
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchiveConfig_With_NotExists_Tag,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceNotExists("data.sakuracloud_archive.foobar"),
				),
				Destroy: true,
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchive_NameSelector_NotExists,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceNotExists("data.sakuracloud_archive.foobar"),
				),
				Destroy: true,
			},
			{
				Config: testAccCheckSakuraCloudDataSourceArchive_TagSelector_NotExists,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckSakuraCloudArchiveDataSourceNotExists("data.sakuracloud_archive.foobar"),
				),
				Destroy: true,
			},
		},
	})
}

func testAccCheckSakuraCloudArchiveDataSourceID(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Can't find Archive data source: %s", n)
		}

		if rs.Primary.ID == "" {
			return errors.New("Archive data source ID not set")
		}
		return nil
	}
}

func testAccCheckSakuraCloudArchiveDataSourceNotExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		_, ok := s.RootModule().Resources[n]
		if ok {
			return fmt.Errorf("Found Archive data source: %s", n)
		}
		return nil
	}
}

func testAccCheckSakuraCloudArchiveDataSourceDestroy(s *terraform.State) error {
	client := testAccProvider.Meta().(*APIClient)
	originalZone := client.Zone
	client.Zone = "tk1v"
	defer func() { client.Zone = originalZone }()

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "sakuracloud_archive" {
			continue
		}

		if rs.Primary.ID == "" {
			continue
		}

		_, err := client.Archive.Read(toSakuraCloudID(rs.Primary.ID))

		if err == nil {
			return errors.New("Archive still exists")
		}
	}

	return nil
}

var testAccCheckSakuraCloudDataSourceArchiveConfig = `
data "sakuracloud_archive" "foobar" {
    filter = {
	name = "Name"
	values = ["Ubuntu Server 16"]
    }
    zone = "tk1v"
}`

var testAccCheckSakuraCloudDataSourceArchiveConfig_With_Tag = `
data "sakuracloud_archive" "foobar" {
    filter = {
	name = "Tags"
	values = ["distro-ubuntu","os-linux"]
    }
    zone = "tk1v"
}`

var testAccCheckSakuraCloudDataSourceArchiveConfig_With_NotExists_Tag = `
data "sakuracloud_archive" "foobar" {
    filter = {
	name = "Tags"
	values = ["distro-ubuntu-xxxxxxxxxxx","os-linux-xxxxxxxx"]
    }
    zone = "tk1v"
}`

var testAccCheckSakuraCloudDataSourceArchiveConfig_NotExists = `
data "sakuracloud_archive" "foobar" {
    filter = {
	name = "Name"
	values = ["xxxxxxxxxxxxxxxxxx"]
    }
    zone = "tk1v"
}`

var testAccCheckSakuraCloudDataSourceArchive_OSType = `
data "sakuracloud_archive" "foobar" {
    os_type = "rancheros"
    zone = "tk1v"
}
`

var testAccCheckSakuraCloudDataSourceArchive_NameSelector_Exists = `
data "sakuracloud_archive" "foobar" {
    name_selectors = ["Ubuntu", "Server","16"]
    zone = "tk1v"
}
`
var testAccCheckSakuraCloudDataSourceArchive_NameSelector_NotExists = `
data "sakuracloud_archive" "foobar" {
    name_selectors = ["xxxxxxxxxx"]
    zone = "tk1v"
}
`

var testAccCheckSakuraCloudDataSourceArchive_TagSelector_Exists = `
data "sakuracloud_archive" "foobar" {
	tag_selectors = ["distro-ubuntu","os-linux"]
    zone = "tk1v"
}`

var testAccCheckSakuraCloudDataSourceArchive_TagSelector_NotExists = `
data "sakuracloud_archive" "foobar" {
	tag_selectors = ["xxxxxxxxxx"]
    zone = "tk1v"
}`
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
)

// Lots of the code here are learnt from the http package

type Proxy struct {
	addr string // listen address
}

type clientConn struct {
	keepAlive bool
	buf       *bufio.ReadWriter
	netconn   net.Conn // connection to the proxy client
	// TODO is it possible that one proxy connection is used to server all the client request?
	// Make things simple at this moment and disable http request keep-alive
	// srvconn net.Conn // connection to the server
}

type ProxyError struct {
	msg string
}

func (pe *ProxyError) Error() string { return pe.msg }

func newProxyError(msg string, err error) *ProxyError {
	return &ProxyError{fmt.Sprintln(msg, err)}
}

func NewProxy(addr string) (proxy *Proxy) {
	proxy = &Proxy{addr: addr}
	return
}

func (py *Proxy) Serve() {
	ln, err := net.Listen("tcp", py.addr)
	if err != nil {
		log.Println("Server creation failed:", err)
		os.Exit(1)
	}
	info.Println("COW proxy listening", py.addr)

	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Println("Client connection:", err)
			continue
		}
		info.Println("New Client:", conn.RemoteAddr())

		c := newClientConn(conn)
		go c.serve()
	}
}

func newClientConn(rwc net.Conn) (c *clientConn) {
	c = &clientConn{netconn: rwc}
	// http pkg uses io.LimitReader with no limit to create a reader, why?
	br := bufio.NewReader(rwc)
	bw := bufio.NewWriter(rwc)
	c.buf = bufio.NewReadWriter(br, bw)
	return
}

func (c *clientConn) close() {
	if c.buf != nil {
		c.buf.Flush()
		c.buf = nil
	}
	if c.netconn != nil {
		info.Printf("Client %v connection closed\n", c.netconn.RemoteAddr())
		c.netconn.Close()
		c.netconn = nil
	}
}

func (c *clientConn) serve() {
	defer c.close()
	var r *Request
	var err error
	for {
		if r, err = parseRequest(c.buf.Reader); err != nil {
			// io.EOF means the client connection is closed
			if err != io.EOF {
				log.Println("Reading client request", err)
			}
			return
		}
		if debug {
			debug.Printf("%v", r)
		} else {
			info.Println(r)
		}

		// TODO Need to do the request in a goroutine to support pipelining?
		// If so, how to maintain the order of finishing request?
		// Consider pipelining later as this is just performance improvement.
		if err = c.doRequest(r); err != nil {
			log.Println("Doing request %s %s:", r.Method, r.URL, err)
			// TODO Should server connection error close client connection?
			// Possible error:
			// 1. the proxy can't find the host
			// 2. broken pipe to the client
			break
		}

		// How to detect closed client connection?
		// Reading client connection will encounter EOF and detect that the
		// connection has been closed.

		// Firefox will create 6 persistent connections to the proxy server.
		// If opening many connections is not a problem, then nothing need
		// to be done.
		// Otherwise, set a read time out and close connection upon timeout.
		// This should not cause problem as
		// 1. I didn't see any independent message sent by firefox in order to
		//    close a persistent connection
		// 2. Sending Connection: Keep-Alive but actually closing the
		//    connection cause no problem for firefox. (The client should be
		//    able to detect closed connection and open a new one.)
		if !r.KeepAlive {
			break
		}
	}
}

func (c *clientConn) doRequest(r *Request) (err error) {
	host := r.URL.Host
	if !hostHasPort(host) {
		host += ":80"
	}
	srvconn, err := net.Dial("tcp", host)
	if err != nil {
		// TODO Find a way report no host error to client. Send back web page?
		// It's weird here, sometimes nslookup can finding host, but net.Dial
		// can't
		return newProxyError("Connecting to: "+host, err)
	}
	// TODO revisit here when implementing keep-alive
	defer srvconn.Close()
	debug.Printf("Connected to %s\n", r.URL.Host)

	// Send request to the server
	if _, err = srvconn.Write(r.raw.Bytes()); err != nil {
		return
	}
	// Send request body
	if r.Method == "POST" {
		srvWriter := bufio.NewWriter(srvconn)
		if err = sendBody(srvWriter, c.buf.Reader, r.Chunking, r.ContLen); err != nil {
			return newProxyError("Sending request body", err)
		}
	}

	// Read server reply
	srvReader := bufio.NewReader(srvconn)
	rp, err := parseResponse(srvReader, r.Method)
	if err != nil {
		return
	}
	c.buf.WriteString(rp.raw.String())
	// Flush response header to the client ASAP
	if err = c.buf.Flush(); err != nil {
		return newProxyError("Flushing response header to client", err)
	}

	// Wrap inside if to avoid function argument evaluation. Would this work?
	if debug {
		debug.Printf("[Response] %s %v\n%v", r.Method, r.URL, rp)
	}

	if rp.HasBody {
		if err = sendBody(c.buf.Writer, srvReader, rp.Chunking, rp.ContLen); err != nil {
			return
		}
	}
	debug.Printf("Finished request %s %s\n", r.Method, r.URL)
	return
}

// Send response body if header specifies content length
func sendBodyWithContLen(w *bufio.Writer, r *bufio.Reader, contLen int64) (err error) {
	debug.Printf("Sending body with content length %d\n", contLen)
	if contLen == 0 {
		return
	}
	// CopyN will copy n bytes unless there's error of EOF. For EOF, it means
	// the connection is closed, return will propagate till serv function and
	// close client connection.
	if _, err = io.CopyN(w, r, contLen); err != nil {
		return newProxyError("Sending response body to client", err)
	}
	return
}

// Send response body if header specifies chunked encoding
func sendBodyChunked(w *bufio.Writer, r *bufio.Reader) (err error) {
	debug.Printf("Sending chunked body\n")

	for {
		var s string
		// Read chunk size line, ignore chunk extension if any
		if s, err = ReadLine(r); err != nil {
			return newProxyError("Reading chunk size", err)
		}
		// debug.Printf("chunk size line %s", s)
		f := strings.SplitN(s, ";", 2)
		var size int64
		if size, err = strconv.ParseInt(f[0], 16, 64); err != nil {
			return newProxyError("Chunk size not valid", err)
		}
		w.WriteString(s)
		w.WriteString("\r\n")

		if size == 0 { // end of chunked data, ignore any trailers
			goto END
		}

		// Read chunk data and send to client
		if _, err = io.CopyN(w, r, size); err != nil {
			return newProxyError("Reading chunked data from server", err)
		}
	END:
		// XXX maybe this kind of error handling should be passed to the
		// client? But if the proxy doesn't know when to stop reading from the
		// server, the only way to avoid blocked reading is to set read time
		// out on server connection. Would that be easier?
		if err = readCheckCRLF(r); err != nil {
			return newProxyError("Reading chunked data CRLF", err)
		}
		w.WriteString("\r\n")
	}
	return
}

// Send message body
func sendBody(w *bufio.Writer, r *bufio.Reader, chunk bool, contLen int64) (err error) {
	if chunk {
		err = sendBodyChunked(w, r)
	} else if contLen >= 0 {
		err = sendBodyWithContLen(w, r, contLen)
	} else {
		// Maybe because this is an HTTP/1.0 server. Just read and wait connection close
		info.Printf("Can't determine body length and not chunked encoding\n")
		if _, err = io.Copy(w, r); err != nil {
			return
		}
	}

	if err = w.Flush(); err != nil {
		return newProxyError("Flushing body to client", err)
	}
	return
}
<commit_msg>Comment: handle client request without response header parsing.<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
)

// Lots of the code here are learnt from the http package

type Proxy struct {
	addr string // listen address
}

type clientConn struct {
	keepAlive bool
	buf       *bufio.ReadWriter
	netconn   net.Conn // connection to the proxy client
	// TODO is it possible that one proxy connection is used to server all the client request?
	// Make things simple at this moment and disable http request keep-alive
	// srvconn net.Conn // connection to the server
}

type ProxyError struct {
	msg string
}

func (pe *ProxyError) Error() string { return pe.msg }

func newProxyError(msg string, err error) *ProxyError {
	return &ProxyError{fmt.Sprintln(msg, err)}
}

func NewProxy(addr string) (proxy *Proxy) {
	proxy = &Proxy{addr: addr}
	return
}

func (py *Proxy) Serve() {
	ln, err := net.Listen("tcp", py.addr)
	if err != nil {
		log.Println("Server creation failed:", err)
		os.Exit(1)
	}
	info.Println("COW proxy listening", py.addr)

	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Println("Client connection:", err)
			continue
		}
		info.Println("New Client:", conn.RemoteAddr())

		c := newClientConn(conn)
		go c.serve()
	}
}

func newClientConn(rwc net.Conn) (c *clientConn) {
	c = &clientConn{netconn: rwc}
	// http pkg uses io.LimitReader with no limit to create a reader, why?
	br := bufio.NewReader(rwc)
	bw := bufio.NewWriter(rwc)
	c.buf = bufio.NewReadWriter(br, bw)
	return
}

func (c *clientConn) close() {
	if c.buf != nil {
		c.buf.Flush()
		c.buf = nil
	}
	if c.netconn != nil {
		info.Printf("Client %v connection closed\n", c.netconn.RemoteAddr())
		c.netconn.Close()
		c.netconn = nil
	}
}

func (c *clientConn) serve() {
	defer c.close()
	var r *Request
	var err error

	// The current implementation serves client request one by one. For each
	// request:
	// 1. Parse client HTTP request
	// 2. Connect to the server and send the request, send the response
	//    back to the client
	// We need to know whether a response is finished so we can start to serve
	// another request. That's why we need to parse content-length header and
	// chunked encoding.

	// TODO If we are going to handle client request using 2 goroutines, one
	// to receive client requests and send it to the web server, the other
	// goroutine just pass data received from the server to the client, then
	// we don't need to parse the response. But persistent connection may
	// require response header parsing and thus don't allow this response pass
	// through.

	// The code for content-length and chunked encoding parsing will always be
	// necessary because http request sent to proxy is different from the one
	// directly sent to web server, the proxy needs to construct the http
	// request.

	for {
		if r, err = parseRequest(c.buf.Reader); err != nil {
			// io.EOF means the client connection is closed
			if err != io.EOF {
				log.Println("Reading client request", err)
			}
			return
		}
		if debug {
			debug.Printf("%v", r)
		} else {
			info.Println(r)
		}

		// TODO Need to do the request in a goroutine to support pipelining?
		// If so, how to maintain the order of finishing request?
		// Consider pipelining later as this is just performance improvement.
		if err = c.doRequest(r); err != nil {
			log.Println("Doing request %s %s:", r.Method, r.URL, err)
			// TODO Should server connection error close client connection?
			// Possible error:
			// 1. the proxy can't find the host
			// 2. broken pipe to the client
			break
		}

		// How to detect closed client connection?
		// Reading client connection will encounter EOF and detect that the
		// connection has been closed.

		// Firefox will create 6 persistent connections to the proxy server.
		// If opening many connections is not a problem, then nothing need
		// to be done.
		// Otherwise, set a read time out and close connection upon timeout.
		// This should not cause problem as
		// 1. I didn't see any independent message sent by firefox in order to
		//    close a persistent connection
		// 2. Sending Connection: Keep-Alive but actually closing the
		//    connection cause no problem for firefox. (The client should be
		//    able to detect closed connection and open a new one.)
		if !r.KeepAlive {
			break
		}
	}
}

func (c *clientConn) doRequest(r *Request) (err error) {
	host := r.URL.Host
	if !hostHasPort(host) {
		host += ":80"
	}
	srvconn, err := net.Dial("tcp", host)
	if err != nil {
		// TODO Find a way report no host error to client. Send back web page?
		// It's weird here, sometimes nslookup can finding host, but net.Dial
		// can't
		return newProxyError("Connecting to: "+host, err)
	}
	// TODO revisit here when implementing keep-alive
	defer srvconn.Close()
	debug.Printf("Connected to %s\n", r.URL.Host)

	// Send request to the server
	if _, err = srvconn.Write(r.raw.Bytes()); err != nil {
		return
	}
	// Send request body
	if r.Method == "POST" {
		srvWriter := bufio.NewWriter(srvconn)
		if err = sendBody(srvWriter, c.buf.Reader, r.Chunking, r.ContLen); err != nil {
			return newProxyError("Sending request body", err)
		}
	}

	// Read server reply
	srvReader := bufio.NewReader(srvconn)
	rp, err := parseResponse(srvReader, r.Method)
	if err != nil {
		return
	}
	c.buf.WriteString(rp.raw.String())
	// Flush response header to the client ASAP
	if err = c.buf.Flush(); err != nil {
		return newProxyError("Flushing response header to client", err)
	}

	// Wrap inside if to avoid function argument evaluation. Would this work?
	if debug {
		debug.Printf("[Response] %s %v\n%v", r.Method, r.URL, rp)
	}

	if rp.HasBody {
		if err = sendBody(c.buf.Writer, srvReader, rp.Chunking, rp.ContLen); err != nil {
			return
		}
	}
	debug.Printf("Finished request %s %s\n", r.Method, r.URL)
	return
}

// Send response body if header specifies content length
func sendBodyWithContLen(w *bufio.Writer, r *bufio.Reader, contLen int64) (err error) {
	debug.Printf("Sending body with content length %d\n", contLen)
	if contLen == 0 {
		return
	}
	// CopyN will copy n bytes unless there's error of EOF. For EOF, it means
	// the connection is closed, return will propagate till serv function and
	// close client connection.
	if _, err = io.CopyN(w, r, contLen); err != nil {
		return newProxyError("Sending response body to client", err)
	}
	return
}

// Send response body if header specifies chunked encoding
func sendBodyChunked(w *bufio.Writer, r *bufio.Reader) (err error) {
	debug.Printf("Sending chunked body\n")

	for {
		var s string
		// Read chunk size line, ignore chunk extension if any
		if s, err = ReadLine(r); err != nil {
			return newProxyError("Reading chunk size", err)
		}
		// debug.Printf("chunk size line %s", s)
		f := strings.SplitN(s, ";", 2)
		var size int64
		if size, err = strconv.ParseInt(f[0], 16, 64); err != nil {
			return newProxyError("Chunk size not valid", err)
		}
		w.WriteString(s)
		w.WriteString("\r\n")

		if size == 0 { // end of chunked data, ignore any trailers
			goto END
		}

		// Read chunk data and send to client
		if _, err = io.CopyN(w, r, size); err != nil {
			return newProxyError("Reading chunked data from server", err)
		}
	END:
		// XXX maybe this kind of error handling should be passed to the
		// client? But if the proxy doesn't know when to stop reading from the
		// server, the only way to avoid blocked reading is to set read time
		// out on server connection. Would that be easier?
		if err = readCheckCRLF(r); err != nil {
			return newProxyError("Reading chunked data CRLF", err)
		}
		w.WriteString("\r\n")
	}
	return
}

// Send message body
func sendBody(w *bufio.Writer, r *bufio.Reader, chunk bool, contLen int64) (err error) {
	if chunk {
		err = sendBodyChunked(w, r)
	} else if contLen >= 0 {
		err = sendBodyWithContLen(w, r, contLen)
	} else {
		// Maybe because this is an HTTP/1.0 server. Just read and wait connection close
		info.Printf("Can't determine body length and not chunked encoding\n")
		if _, err = io.Copy(w, r); err != nil {
			return
		}
	}

	if err = w.Flush(); err != nil {
		return newProxyError("Flushing body to client", err)
	}
	return
}
<|endoftext|>


<commit_before>package svgutil

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/rustyoz/svg"
)

func ReadFile(filename, name string, scale float64) (*svg.Svg, error) {
	// https://godoc.org/github.com/rustyoz/svg#Svg
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	name = strings.TrimSpace(name)
	if len(name) == 0 {
		name = filename
	}

	return svg.ParseSvgFromReader(file, filename, scale)
}

func AspectRatio(img *svg.Svg) (float64, error) {
	if img == nil {
		return 0, errors.New("E_NO_IMAGE")
	}
	vals, err := img.ViewBoxValues()
	if err != nil {
		return 0, err
	}
	if len(vals) == 2 {
		if vals[0] == 0 {
			return 0, errors.New("E_DENOM_HEIGHT==0")
		}
		return vals[1] / vals[0], nil
	} else if len(vals) == 4 {
		w := vals[2] - vals[0]
		h := vals[3] - vals[1]
		if h == 0 {
			return 0, errors.New("E_DENOM_HEIGHT==0")
		}
		return w / h, nil
	}
	return 0, fmt.Errorf("E_BAD_VIEWBOX_LEN [%v]", len(vals))
}
<commit_msg>fix: image/svgutil: ineffassign<commit_after>package svgutil

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/rustyoz/svg"
)

func ReadFile(filename, name string, scale float64) (*svg.Svg, error) {
	// https://godoc.org/github.com/rustyoz/svg#Svg
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	name = strings.TrimSpace(name)
	if len(name) == 0 {
		name = filename
	}

	return svg.ParseSvgFromReader(file, name, scale)
}

func AspectRatio(img *svg.Svg) (float64, error) {
	if img == nil {
		return 0, errors.New("E_NO_IMAGE")
	}
	vals, err := img.ViewBoxValues()
	if err != nil {
		return 0, err
	}
	if len(vals) == 2 {
		if vals[0] == 0 {
			return 0, errors.New("E_DENOM_HEIGHT==0")
		}
		return vals[1] / vals[0], nil
	} else if len(vals) == 4 {
		w := vals[2] - vals[0]
		h := vals[3] - vals[1]
		if h == 0 {
			return 0, errors.New("E_DENOM_HEIGHT==0")
		}
		return w / h, nil
	}
	return 0, fmt.Errorf("E_BAD_VIEWBOX_LEN [%v]", len(vals))
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package blockfile provides methods for reading packets from blockfiles
// generated by stenotype.
package blockfile

import (
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
	"time"
	"unsafe"

	"github.com/google/gopacket"
	"github.com/google/stenographer/base"
	"github.com/google/stenographer/indexfile"
	"github.com/google/stenographer/query"
	"github.com/google/stenographer/stats"
	"golang.org/x/net/context"
)

// #include <linux/if_packet.h>
import "C"

var (
	v                = base.V // Verbose logging
	packetReadNanos  = stats.S.Get("packet_read_nanos")
	packetScanNanos  = stats.S.Get("packet_scan_nanos")
	packetsRead      = stats.S.Get("packets_read")
	packetsScanned   = stats.S.Get("packets_scanned")
	packetBlocksRead = stats.S.Get("packets_blocks_read")
)

// BlockFile provides an interface to a single stenotype file on disk and its
// associated index.
type BlockFile struct {
	name string
	f    *os.File
	i    *indexfile.IndexFile
	mu   sync.RWMutex // Stops Close() from invalidating a file before a current query is done with it.
	done chan struct{}
}

// NewBlockFile opens up a named block file (and its index), returning a handle
// which can be used to look up packets.
func NewBlockFile(filename string) (*BlockFile, error) {
	v(1, "Blockfile opening: %q", filename)
	f, err := os.Open(filename)
	if err != nil {
		return nil, fmt.Errorf("could not open %q: %v", filename, err)
	}
	i, err := indexfile.NewIndexFile(indexfile.IndexPathFromBlockfilePath(filename))
	if err != nil {
		f.Close()
		return nil, fmt.Errorf("could not open index for %q: %v", filename, err)
	}
	return &BlockFile{
		f:    f,
		i:    i,
		name: filename,
		done: make(chan struct{}),
	}, nil
}

// Name returns the name of the file underlying this blockfile.
func (b *BlockFile) Name() string {
	return b.name
}

// readPacket reads a single packet from the file at the given position.
// It updates the passed in CaptureInfo with information on the packet.
func (b *BlockFile) readPacket(pos int64, ci *gopacket.CaptureInfo) ([]byte, error) {
	// 28 bytes actually isn't the entire packet header, but it's all the fields
	// that we care about.
	packetsRead.Increment()
	defer packetReadNanos.NanoTimer()()
	var dataBuf [28]byte
	_, err := b.f.ReadAt(dataBuf[:], pos)
	if err != nil {
		return nil, err
	}
	pkt := (*C.struct_tpacket3_hdr)(unsafe.Pointer(&dataBuf[0]))
	*ci = gopacket.CaptureInfo{
		Timestamp:     time.Unix(int64(pkt.tp_sec), int64(pkt.tp_nsec)),
		Length:        int(pkt.tp_len),
		CaptureLength: int(pkt.tp_snaplen),
	}
	out := make([]byte, ci.CaptureLength)
	pos += int64(pkt.tp_mac)
	_, err = b.f.ReadAt(out, pos)
	return out, err
}

// Close cleans up this blockfile.
func (b *BlockFile) Close() (err error) {
	v(2, "Blockfile closing: %q", b.name)
	close(b.done)
	b.mu.Lock()
	defer b.mu.Unlock()
	v(3, "Blockfile closing file descriptors: %q", b.name)
	if e := b.i.Close(); e != nil {
		err = e
	}
	if e := b.f.Close(); e != nil {
		err = e
	}
	b.i, b.f = nil, nil
	return
}

// allPacketsIter implements Iter.
type allPacketsIter struct {
	*BlockFile
	blockData        [1 << 20]byte
	block            *C.struct_tpacket_hdr_v1
	pkt              *C.struct_tpacket3_hdr
	blockPacketsRead int
	blockOffset      int64
	packetOffset     int // offset of packet in block
	err              error
	done             bool
}

func (a *allPacketsIter) Next() bool {
	defer packetScanNanos.NanoTimer()()
	if a.err != nil || a.done {
		return false
	}
	for a.block == nil || a.blockPacketsRead == int(a.block.num_pkts) {
		packetBlocksRead.Increment()
		_, err := a.f.ReadAt(a.blockData[:], a.blockOffset)
		if err == io.EOF {
			a.done = true
			return false
		} else if err != nil {
			a.err = fmt.Errorf("could not read block at %v: %v", a.blockOffset, err)
			return false
		}
		baseHdr := (*C.struct_tpacket_block_desc)(unsafe.Pointer(&a.blockData[0]))
		a.block = (*C.struct_tpacket_hdr_v1)(unsafe.Pointer(&baseHdr.hdr[0]))
		a.blockOffset += 1 << 20
		a.blockPacketsRead = 0
		a.pkt = nil
	}
	a.blockPacketsRead++
	if a.pkt == nil {
		a.packetOffset = int(a.block.offset_to_first_pkt)
	} else if a.pkt.tp_next_offset != 0 {
		a.packetOffset += int(a.pkt.tp_next_offset)
	} else {
		a.err = errors.New("block format currently not supported")
		return false
	}
	a.pkt = (*C.struct_tpacket3_hdr)(unsafe.Pointer(&a.blockData[a.packetOffset]))
	packetsScanned.Increment()
	return true
}

func (a *allPacketsIter) Packet() *base.Packet {
	start := a.packetOffset + int(a.pkt.tp_mac)
	buf := a.blockData[start : start+int(a.pkt.tp_snaplen)]
	p := &base.Packet{Data: buf}
	p.CaptureInfo.Timestamp = time.Unix(int64(a.pkt.tp_sec), int64(a.pkt.tp_nsec))
	p.CaptureInfo.Length = int(a.pkt.tp_len)
	p.CaptureInfo.CaptureLength = int(a.pkt.tp_snaplen)
	return p
}

func (a *allPacketsIter) Err() error {
	return a.err
}

// AllPackets returns a packet channel to which all packets in the blockfile are
// sent.
func (b *BlockFile) AllPackets() *base.PacketChan {
	b.mu.RLock()
	c := base.NewPacketChan(100)
	go func() {
		defer b.mu.RUnlock()
		pkts := &allPacketsIter{BlockFile: b}
		for pkts.Next() {
			c.Send(pkts.Packet())
		}
		c.Close(pkts.Err())
	}()
	return c
}

// Positions returns the positions in the blockfile of all packets matched by
// the passed-in query.
func (b *BlockFile) Positions(ctx context.Context, q query.Query) (base.Positions, error) {
	b.mu.RLock()
	defer b.mu.RUnlock()
	return b.positionsLocked(ctx, q)
}

// positionsLocked returns the positions in the blockfile of all packets matched by
// the passed-in query.  b.mu must be locked.
func (b *BlockFile) positionsLocked(ctx context.Context, q query.Query) (base.Positions, error) {
	if b.i == nil || b.f == nil {
		// If we're closed, just return nothing.
		return nil, nil
	}
	return q.LookupIn(ctx, b.i)
}

// Lookup returns all packets in the blockfile matched by the passed-in query.
func (b *BlockFile) Lookup(ctx context.Context, q query.Query, out *base.PacketChan) {
	b.mu.RLock()
	defer b.mu.RUnlock()

	var ci gopacket.CaptureInfo
	v(2, "Blockfile %q looking up query %q", q.String(), b.name)
	start := time.Now()
	positions, err := b.positionsLocked(ctx, q)
	if err != nil {
		out.Close(fmt.Errorf("index lookup failure: %v", err))
		return
	}
	if positions.IsAllPositions() {
		v(2, "Blockfile %q reading all packets", b.name)
		iter := &allPacketsIter{BlockFile: b}
	all_packets_loop:
		for iter.Next() {
			select {
			case <-ctx.Done():
				v(2, "Blockfile %q canceling packet read", b.name)
				break all_packets_loop
			case <-b.done:
				v(2, "Blockfile %q closing, breaking out of query", b.name)
				break all_packets_loop
			case out.C <- iter.Packet():
			}
		}
		if iter.Err() != nil {
			out.Close(fmt.Errorf("error reading all packets from %q: %v", b.name, iter.Err()))
			return
		}
	} else {
		v(2, "Blockfile %q reading %v packets", b.name, len(positions))
	query_packets_loop:
		for _, pos := range positions {
			buffer, err := b.readPacket(pos, &ci)
			if err != nil {
				v(2, "Blockfile %q error reading packet: %v", b.name, err)
				out.Close(fmt.Errorf("error reading packets from %q @ %v: %v", b.name, pos, err))
				return
			}
			select {
			case <-ctx.Done():
				v(2, "Blockfile %q canceling packet read", b.name)
				break query_packets_loop
			case <-b.done:
				v(2, "Blockfile %q closing, breaking out of query", b.name)
				break query_packets_loop
			case out.C <- &base.Packet{Data: buffer, CaptureInfo: ci}:
			}
		}
	}
	v(2, "Blockfile %q finished reading all packets in %v", b.name, time.Since(start))
	out.Close(ctx.Err())
}

// DumpIndex dumps out a "human-readable" debug version of the blockfile's index
// to the given writer.
func (b *BlockFile) DumpIndex(out io.Writer, start, finish []byte) {
	b.mu.RLock()
	defer b.mu.RUnlock()
	b.i.Dump(out, start, finish)
}
<commit_msg>blockfile: Fix overwriting 'blockData' before packet is read.<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package blockfile provides methods for reading packets from blockfiles
// generated by stenotype.
package blockfile

import (
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
	"time"
	"unsafe"

	"github.com/google/gopacket"
	"github.com/google/stenographer/base"
	"github.com/google/stenographer/indexfile"
	"github.com/google/stenographer/query"
	"github.com/google/stenographer/stats"
	"golang.org/x/net/context"
)

// #include <linux/if_packet.h>
import "C"

var (
	v                = base.V // Verbose logging
	packetReadNanos  = stats.S.Get("packet_read_nanos")
	packetScanNanos  = stats.S.Get("packet_scan_nanos")
	packetsRead      = stats.S.Get("packets_read")
	packetsScanned   = stats.S.Get("packets_scanned")
	packetBlocksRead = stats.S.Get("packets_blocks_read")
)

// BlockFile provides an interface to a single stenotype file on disk and its
// associated index.
type BlockFile struct {
	name string
	f    *os.File
	i    *indexfile.IndexFile
	mu   sync.RWMutex // Stops Close() from invalidating a file before a current query is done with it.
	done chan struct{}
}

// NewBlockFile opens up a named block file (and its index), returning a handle
// which can be used to look up packets.
func NewBlockFile(filename string) (*BlockFile, error) {
	v(1, "Blockfile opening: %q", filename)
	f, err := os.Open(filename)
	if err != nil {
		return nil, fmt.Errorf("could not open %q: %v", filename, err)
	}
	i, err := indexfile.NewIndexFile(indexfile.IndexPathFromBlockfilePath(filename))
	if err != nil {
		f.Close()
		return nil, fmt.Errorf("could not open index for %q: %v", filename, err)
	}
	return &BlockFile{
		f:    f,
		i:    i,
		name: filename,
		done: make(chan struct{}),
	}, nil
}

// Name returns the name of the file underlying this blockfile.
func (b *BlockFile) Name() string {
	return b.name
}

// readPacket reads a single packet from the file at the given position.
// It updates the passed in CaptureInfo with information on the packet.
func (b *BlockFile) readPacket(pos int64, ci *gopacket.CaptureInfo) ([]byte, error) {
	// 28 bytes actually isn't the entire packet header, but it's all the fields
	// that we care about.
	packetsRead.Increment()
	defer packetReadNanos.NanoTimer()()
	var dataBuf [28]byte
	_, err := b.f.ReadAt(dataBuf[:], pos)
	if err != nil {
		return nil, err
	}
	pkt := (*C.struct_tpacket3_hdr)(unsafe.Pointer(&dataBuf[0]))
	*ci = gopacket.CaptureInfo{
		Timestamp:     time.Unix(int64(pkt.tp_sec), int64(pkt.tp_nsec)),
		Length:        int(pkt.tp_len),
		CaptureLength: int(pkt.tp_snaplen),
	}
	out := make([]byte, ci.CaptureLength)
	pos += int64(pkt.tp_mac)
	_, err = b.f.ReadAt(out, pos)
	return out, err
}

// Close cleans up this blockfile.
func (b *BlockFile) Close() (err error) {
	v(2, "Blockfile closing: %q", b.name)
	close(b.done)
	b.mu.Lock()
	defer b.mu.Unlock()
	v(3, "Blockfile closing file descriptors: %q", b.name)
	if e := b.i.Close(); e != nil {
		err = e
	}
	if e := b.f.Close(); e != nil {
		err = e
	}
	b.i, b.f = nil, nil
	return
}

// allPacketsIter implements Iter.
type allPacketsIter struct {
	*BlockFile
	blockData        []byte
	block            *C.struct_tpacket_hdr_v1
	pkt              *C.struct_tpacket3_hdr
	blockPacketsRead int
	blockOffset      int64
	packetOffset     int // offset of packet in block
	err              error
	done             bool
}

func (a *allPacketsIter) Next() bool {
	defer packetScanNanos.NanoTimer()()
	if a.err != nil || a.done {
		return false
	}
	for a.block == nil || a.blockPacketsRead == int(a.block.num_pkts) {
		packetBlocksRead.Increment()
		a.blockData = make([]byte, 1 << 20)
		_, err := a.f.ReadAt(a.blockData[:], a.blockOffset)
		if err == io.EOF {
			a.done = true
			return false
		} else if err != nil {
			a.err = fmt.Errorf("could not read block at %v: %v", a.blockOffset, err)
			return false
		}
		baseHdr := (*C.struct_tpacket_block_desc)(unsafe.Pointer(&a.blockData[0]))
		a.block = (*C.struct_tpacket_hdr_v1)(unsafe.Pointer(&baseHdr.hdr[0]))
		a.blockOffset += 1 << 20
		a.blockPacketsRead = 0
		a.pkt = nil
	}
	a.blockPacketsRead++
	if a.pkt == nil {
		a.packetOffset = int(a.block.offset_to_first_pkt)
	} else if a.pkt.tp_next_offset != 0 {
		a.packetOffset += int(a.pkt.tp_next_offset)
	} else {
		a.err = errors.New("block format currently not supported")
		return false
	}
	a.pkt = (*C.struct_tpacket3_hdr)(unsafe.Pointer(&a.blockData[a.packetOffset]))
	packetsScanned.Increment()
	return true
}

func (a *allPacketsIter) Packet() *base.Packet {
	start := a.packetOffset + int(a.pkt.tp_mac)
	buf := a.blockData[start : start+int(a.pkt.tp_snaplen)]
	p := &base.Packet{Data: buf}
	p.CaptureInfo.Timestamp = time.Unix(int64(a.pkt.tp_sec), int64(a.pkt.tp_nsec))
	p.CaptureInfo.Length = int(a.pkt.tp_len)
	p.CaptureInfo.CaptureLength = int(a.pkt.tp_snaplen)
	return p
}

func (a *allPacketsIter) Err() error {
	return a.err
}

// AllPackets returns a packet channel to which all packets in the blockfile are
// sent.
func (b *BlockFile) AllPackets() *base.PacketChan {
	b.mu.RLock()
	c := base.NewPacketChan(100)
	go func() {
		defer b.mu.RUnlock()
		pkts := &allPacketsIter{BlockFile: b}
		for pkts.Next() {
			c.Send(pkts.Packet())
		}
		c.Close(pkts.Err())
	}()
	return c
}

// Positions returns the positions in the blockfile of all packets matched by
// the passed-in query.
func (b *BlockFile) Positions(ctx context.Context, q query.Query) (base.Positions, error) {
	b.mu.RLock()
	defer b.mu.RUnlock()
	return b.positionsLocked(ctx, q)
}

// positionsLocked returns the positions in the blockfile of all packets matched by
// the passed-in query.  b.mu must be locked.
func (b *BlockFile) positionsLocked(ctx context.Context, q query.Query) (base.Positions, error) {
	if b.i == nil || b.f == nil {
		// If we're closed, just return nothing.
		return nil, nil
	}
	return q.LookupIn(ctx, b.i)
}

// Lookup returns all packets in the blockfile matched by the passed-in query.
func (b *BlockFile) Lookup(ctx context.Context, q query.Query, out *base.PacketChan) {
	b.mu.RLock()
	defer b.mu.RUnlock()

	var ci gopacket.CaptureInfo
	v(2, "Blockfile %q looking up query %q", q.String(), b.name)
	start := time.Now()
	positions, err := b.positionsLocked(ctx, q)
	if err != nil {
		out.Close(fmt.Errorf("index lookup failure: %v", err))
		return
	}
	if positions.IsAllPositions() {
		v(2, "Blockfile %q reading all packets", b.name)
		iter := &allPacketsIter{BlockFile: b}
	all_packets_loop:
		for iter.Next() {
			select {
			case <-ctx.Done():
				v(2, "Blockfile %q canceling packet read", b.name)
				break all_packets_loop
			case <-b.done:
				v(2, "Blockfile %q closing, breaking out of query", b.name)
				break all_packets_loop
			case out.C <- iter.Packet():
			}
		}
		if iter.Err() != nil {
			out.Close(fmt.Errorf("error reading all packets from %q: %v", b.name, iter.Err()))
			return
		}
	} else {
		v(2, "Blockfile %q reading %v packets", b.name, len(positions))
	query_packets_loop:
		for _, pos := range positions {
			buffer, err := b.readPacket(pos, &ci)
			if err != nil {
				v(2, "Blockfile %q error reading packet: %v", b.name, err)
				out.Close(fmt.Errorf("error reading packets from %q @ %v: %v", b.name, pos, err))
				return
			}
			select {
			case <-ctx.Done():
				v(2, "Blockfile %q canceling packet read", b.name)
				break query_packets_loop
			case <-b.done:
				v(2, "Blockfile %q closing, breaking out of query", b.name)
				break query_packets_loop
			case out.C <- &base.Packet{Data: buffer, CaptureInfo: ci}:
			}
		}
	}
	v(2, "Blockfile %q finished reading all packets in %v", b.name, time.Since(start))
	out.Close(ctx.Err())
}

// DumpIndex dumps out a "human-readable" debug version of the blockfile's index
// to the given writer.
func (b *BlockFile) DumpIndex(out io.Writer, start, finish []byte) {
	b.mu.RLock()
	defer b.mu.RUnlock()
	b.i.Dump(out, start, finish)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
)

type Opts struct {
	path      string
	host      string
	port      int
	noBrowser bool
}

func ParseOpts() (result Opts, err error) {
	var opts Opts
	var help bool

	flag.BoolVar(&help, "h", false, "")
	flag.BoolVar(&help, "help", false, "")
	flag.StringVar(&opts.host, "host", "localhost", "")
	flag.IntVar(&opts.port, "p", 8888, "")
	flag.IntVar(&opts.port, "port", 8888, "")
	flag.BoolVar(&opts.noBrowser, "no-browser", false, "")
	flag.Parse()

	opts.path = flag.Arg(0)
	if opts.path == "" {
		opts.path = "joseki.jkf"
	}

	if help {
		return opts, errors.New(
			`Usage:
    kifu-notebook [options] [JKF_FILE]

Options:
    -h, --help
        Show this help message
    --host HOST
        Host to bind (default: localhost)
    -p PORT, --port PORT
        Port to listen on (default: 8888)
    --no-browser
        Don't open browser`)
	} else {
		return opts, nil
	}
}
<commit_msg>Fix unneeded else<commit_after>package main

import (
	"errors"
	"flag"
)

type Opts struct {
	path      string
	host      string
	port      int
	noBrowser bool
}

func ParseOpts() (result Opts, err error) {
	var opts Opts
	var help bool

	flag.BoolVar(&help, "h", false, "")
	flag.BoolVar(&help, "help", false, "")
	flag.StringVar(&opts.host, "host", "localhost", "")
	flag.IntVar(&opts.port, "p", 8888, "")
	flag.IntVar(&opts.port, "port", 8888, "")
	flag.BoolVar(&opts.noBrowser, "no-browser", false, "")
	flag.Parse()

	opts.path = flag.Arg(0)
	if opts.path == "" {
		opts.path = "joseki.jkf"
	}

	if help {
		return opts, errors.New(
			`Usage:
    kifu-notebook [options] [JKF_FILE]

Options:
    -h, --help
        Show this help message
    --host HOST
        Host to bind (default: localhost)
    -p PORT, --port PORT
        Port to listen on (default: 8888)
    --no-browser
        Don't open browser`)
	}
	return opts, nil
}
<|endoftext|>


<commit_before>/*
An API client for opensubtitles.org

This is a client for the OSDb protocol. Currently the package only allows movie
identification, and subtitles search.
*/
package osdb

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"os"
)

const (
	ChunkSize = 65536 // 64k
)

func Hash(path string) (hash uint64, err error) {
	// Check file size.
	fi, err := os.Stat(path)
	if err != nil {
		return
	}
	if fi.Size() < ChunkSize {
		return 0, fmt.Errorf("File is too small")
	}

	file, err := os.Open(path)
	if err != nil {
		return
	}

	// Read head and tail blocks.
	buf := make([]byte, ChunkSize*2)
	err = readChunk(file, 0, buf[:ChunkSize])
	if err != nil {
		return
	}
	err = readChunk(file, fi.Size()-ChunkSize, buf[ChunkSize:])
	if err != nil {
		return
	}

	// Convert to uint64, and sum.
	var nums [(ChunkSize * 2) / 8]uint64
	reader := bytes.NewReader(buf)
	err = binary.Read(reader, binary.LittleEndian, &nums)
	if err != nil {
		return 0, err
	}
	for _, num := range nums {
		hash += num
	}

	return hash + uint64(fi.Size()), nil
}

// Fill buffer
func readChunk(file *os.File, offset int64, buf []byte) (err error) {
	n, err := file.ReadAt(buf, offset)
	if err != nil {
		return
	}
	if n != ChunkSize {
		return fmt.Errorf("Invalid read", n)
	}
	return
}
<commit_msg>Add basic osdb.SearchForFile<commit_after>/*
An API client for opensubtitles.org

This is a client for the OSDb protocol. Currently the package only allows movie
identification, and subtitles search.
*/
package osdb

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"os"
	"strings"

	"github.com/mattn/go-xmlrpc"
)

const (
	ChunkSize     = 65536 // 64k
	Server        = "http://api.opensubtitles.org/xml-rpc"
	SearchLimit   = 100
	SuccessStatus = "200 OK"
)

var (
	UserAgent = "SubDownloader 2.0.10" // FIXME register a user-agent, one day.
	Token     = ""
)

type Result struct {
}

// Generate a OSDB hash for a file
func Hash(path string) (hash uint64, err error) {
	// Check file size.
	fi, err := os.Stat(path)
	if err != nil {
		return
	}
	if fi.Size() < ChunkSize {
		return 0, fmt.Errorf("File is too small")
	}

	file, err := os.Open(path)
	if err != nil {
		return
	}

	// Read head and tail blocks.
	buf := make([]byte, ChunkSize*2)
	err = readChunk(file, 0, buf[:ChunkSize])
	if err != nil {
		return
	}
	err = readChunk(file, fi.Size()-ChunkSize, buf[ChunkSize:])
	if err != nil {
		return
	}

	// Convert to uint64, and sum.
	var nums [(ChunkSize * 2) / 8]uint64
	reader := bytes.NewReader(buf)
	err = binary.Read(reader, binary.LittleEndian, &nums)
	if err != nil {
		return 0, err
	}
	for _, num := range nums {
		hash += num
	}

	return hash + uint64(fi.Size()), nil
}

// Search subtitles in `languages` for a file at `path`.
func SearchForFile(path string, langs []string) ([]Result, error) {
	// Login anonymously if no token has been set yet.
	if Token == "" {
		tok, err := Login("", "", "")
		Token = tok
		if err != nil {
			return nil, err
		}
	}

	// Get file size
	fi, err := os.Stat(path)
	if err != nil {
		return nil, err
	}
	size := fi.Size()

	// Get file hash
	h, err := Hash(path)
	if err != nil {
		return nil, err
	}

	// Query OSDB....
	res, err := xmlrpc.Call(Server,
		"SearchSubtitles",
		Token,
		[1]map[string]interface{}{ // go-xmlrpc panics on slices...
			map[string]interface{}{
				"moviehash":     fmt.Sprintf("%x", h),
				"moviebytesize": size,
				"sublanguageid": strings.Join(langs, ","),
			},
		},
		map[string]int{"limit": SearchLimit},
	)
	if err != nil {
		return nil, err
	}
	return parseSearchResults(&res)
}

// Untangle the XMLRPC mess, yay.
func parseSearchResults(resp *interface{}) ([]Result, error) {
	res := (*resp).(xmlrpc.Struct)
	if res["status"] != SuccessStatus {
		return nil, fmt.Errorf("Search error: %v", res["status"])
	}

	count := len(res["data"].(xmlrpc.Array))
	results := make([]Result, count)

	// FIXME debug mattn/go-xmlrpc, then
	// FIXME build a slice of Result
	//for _, raw := range res["data"].(xmlrpc.Struct) { }

	return results, nil
}

// Login to the API. Return a token
func Login(user string, pass string, lang string) (string, error) {
	res, err := xmlrpc.Call(Server, "LogIn", user, pass, lang, UserAgent)
	if err != nil {
		return "", err
	}
	data := res.(xmlrpc.Struct)
	if data["status"] != SuccessStatus {
		return "", fmt.Errorf("login error: %s", data["status"])
	}
	return data["token"].(string), nil
}

// Logout
func Logout(tok string) (err error) {
	_, err = xmlrpc.Call(Server, "LogOut", tok)
	return
}

// Read a chunk of a file at `offset` so as to fill `buf`.
func readChunk(file *os.File, offset int64, buf []byte) (err error) {
	n, err := file.ReadAt(buf, offset)
	if err != nil {
		return
	}
	if n != ChunkSize {
		return fmt.Errorf("Invalid read", n)
	}
	return
}
<|endoftext|>


<commit_before>package iching

import "strconv"

func Itoiching(in uint64) string {
	s := make([]rune, 0, 64/6+1)
	for {
		if in == 0 {
			break
		}

		s = append(s, table[in&077])
		in = in >> 6
	}

	if len(s) == 0 {
		s = append(s, table[0])
	}

	for i := 0; i < len(s)/2; i++ {
		s[i], s[len(s)-i-1] = s[len(s)-i-1], s[i]
	}

	return string(s)
}

func Ichingtoi(in string) (uint64, error) {
	r := uint64(0)
	for _, ch := range []rune(in) {
		idx := int(ch - 19904)
		if idx < 0 || idx > len(invtable) {
			return 0, &strconv.NumError{"iching.Ichingtoi", in, strconv.ErrSyntax}
		}
		r = r << 6
		r += invtable[idx]
	}
	return r, nil
}

func QianPad(in string, sz int) string {
	return pad(in, sz, 0x4dc0)
}

func SpacePad(in string, sz int) string {
	return pad(in, sz, 0x20)
}

func pad(in string, sz int, padch rune) string {
	rin := []rune(in)
	if len(rin) >= sz {
		return in
	}

	r := make([]rune, 0, sz)

	for i := 0; i < sz-len(rin); i++ {
		r = append(r, padch)
	}

	for _, ch := range rin {
		r = append(r, ch)
	}

	return string(r)
}
<commit_msg>godoc<commit_after>/*
I Ching or "Classic of Change" is a Chinese book about telling the future or something. It doesn't matter.

What matters is that I Ching defines a set of 64 "figures", called hexagrams, each consisting of 6 stacked horizontal lines, where each line is either a solid, unbroken line (Yang) or a line with a single gap (Yin).

Each hexagram can therefore be easily used to encode a number up to 6bits of entropy, this library implements a function (`Itoiching`) to convert an arbitrary 64bit unsigned integer into its I Ching representation and a function (`Ichingtoi`) to do the reverse.

Each I Ching character represents a 6 bit value, strings are ordered with the most significant 6 bits of the number first. Withing a I Ching character the top line represents the most significant bit of the sextet.
The unbroken line (Yand) corresponds to 0, the broken line corresponds to 1. We do not use the King Wen sequence.
*/
package iching

import "strconv"

// Converts in to its I Ching encoding
func Itoiching(in uint64) string {
	s := make([]rune, 0, 64/6+1)
	for {
		if in == 0 {
			break
		}

		s = append(s, table[in&077])
		in = in >> 6
	}

	if len(s) == 0 {
		s = append(s, table[0])
	}

	for i := 0; i < len(s)/2; i++ {
		s[i], s[len(s)-i-1] = s[len(s)-i-1], s[i]
	}

	return string(s)
}

// Parses a number encoded as I Ching characters
func Ichingtoi(in string) (uint64, error) {
	r := uint64(0)
	for _, ch := range []rune(in) {
		idx := int(ch - 19904)
		if idx < 0 || idx > len(invtable) {
			return 0, &strconv.NumError{"iching.Ichingtoi", in, strconv.ErrSyntax}
		}
		r = r << 6
		r += invtable[idx]
	}
	return r, nil
}

// Pads a string with Qián characters (the character that represents 0 in our encoding) up to a total sz runes
func QianPad(in string, sz int) string {
	return pad(in, sz, 0x4dc0)
}

// Pads a string with space characters up to a total of sz runes
func SpacePad(in string, sz int) string {
	return pad(in, sz, 0x20)
}

func pad(in string, sz int, padch rune) string {
	rin := []rune(in)
	if len(rin) >= sz {
		return in
	}

	r := make([]rune, 0, sz)

	for i := 0; i < sz-len(rin); i++ {
		r = append(r, padch)
	}

	for _, ch := range rin {
		r = append(r, ch)
	}

	return string(r)
}
<|endoftext|>


<commit_before>package api

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"github.com/chai2010/webp"
	"github.com/nfnt/resize"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/kamoljan/sushiobrol/conf"
	"github.com/kamoljan/sushiobrol/json"
)

type iconf struct {
	image                                                image.Image
	machine, format, density, ui, hash, color, fid, path string
	width, height                                        uint
}

func Post(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Write(json.Message("ERROR", "Not supported Method"))
		return
	}
	reader, err := r.MultipartReader()
	if err != nil {
		w.Write(json.Message("ERROR", "Client should support multipart/form-data"))
		return
	}
	buf := bytes.NewBufferString("")
	for {
		part, err := reader.NextPart()
		if err == io.EOF {
			break
		}
		if part.FileName() == "" { // if empty skip this iteration
			continue
		}
		_, err = io.Copy(buf, part)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	defer r.Body.Close()
	var result json.Result
	var ic iconf
	ic.machine = conf.Image.Machine
	ic.hash = fmt.Sprintf("%x", sha1.Sum(buf.Bytes()))
	if conf.InputType == "jpeg" {
		ic.image, _, err = image.Decode(buf)
	} else {
		ic.image, err = webp.Decode(buf)
	}
	if err != nil {
		w.Write(json.Message("ERROR", "Unable to decode your image! Type="+conf.InputType+" error:"+err.Error()))
		return
	}
	setColor(&ic)
	for _, format := range conf.Image.Format { // jpeg, webp, ...
		for _, screen := range conf.Image.Screen {
			ic.format = format
			ic.ui = screen.Ui
			ic.density = screen.Density
			ic.width = screen.Width
			if ic.fid, err = imgToFile(&ic); err != nil {
				w.Write(json.Message("ERROR", "Unable to create a file"))
				return
			}
			fid := json.Fid{fmt.Sprintf("%s_%s", screen.Density, screen.Ui), ic.fid}
			result.Image = append(result.Image, fid)
		}
	}
	w.Write(json.Message("OK", &result))
}

func imgToFile(ic *iconf) (string, error) {
	img := resize.Resize(ic.width, 0, ic.image, resize.NearestNeighbor)
	ic.height = uint(img.Bounds().Size().Y)
	ic.fid = fmt.Sprintf("%s-%s-%s-%s-%s-%s-%d-%d", ic.machine, ic.format, ic.density, ic.ui, ic.hash, ic.color, ic.width, ic.height)
	dir := fmt.Sprintf("%s/%s/%s/%s/%s/%s/%s", conf.SushiobrolStore, ic.machine, ic.format, ic.density, ic.ui, ic.hash[0:2], ic.hash[2:4])
	ic.path = fmt.Sprintf("%s/%s", dir, ic.fid)
	out, err := os.Create(ic.path)
	if err != nil {
		log.Println(err)
		return "", err
	}
	defer out.Close()
	if webp.Encode(out, img, &webp.Options{conf.Lossless, conf.Quality}); err != nil {
		fmt.Println("ERROR: Unable to Encode into webp")
		return "", err
	}
	return ic.fid, err
}

func setColor(ic *iconf) {
	img1x1 := resize.Resize(1, 1, ic.image, resize.NearestNeighbor)
	red, green, blue, _ := img1x1.At(0, 0).RGBA()
	ic.color = fmt.Sprintf("%X%X%X", red>>8, green>>8, blue>>8) // removing 1 byte 9A16->9A
}
<commit_msg>Comments from Tv and dsal: Usage of r.FormFile, TeeReader and sha1. Thank you<commit_after>package api

import (
	"bufio"
	"crypto/sha1"
	"fmt"
	"github.com/chai2010/webp"
	"github.com/nfnt/resize"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/kamoljan/sushiobrol/conf"
	"github.com/kamoljan/sushiobrol/json"
)

type iconf struct {
	image                                                image.Image
	machine, format, density, ui, hash, color, fid, path string
	width, height                                        uint
}

func Post(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Write(json.Message("ERROR", "Not supported Method"))
		return
	}
	f, _, err := r.FormFile("image")
	if err != nil {
		w.Write(json.Message("ERROR", "Can't Find Image"))
		return
	}
	defer f.Close()
	buf := bufio.NewReader(f)
	h := sha1.New()
	var result json.Result
	var ic iconf
	ic.machine = conf.Image.Machine
	if conf.InputType == "jpeg" {
		ic.image, _, err = image.Decode(io.TeeReader(buf, h))
	} else {
		ic.image, err = webp.Decode(io.TeeReader(buf, h))
	}
	ic.hash = fmt.Sprintf("%x", h.Sum(nil))
	if err != nil {
		w.Write(json.Message("ERROR", "Unable to decode your image! Type="+conf.InputType+" error:"+err.Error()))
		return
	}
	setColor(&ic)
	for _, format := range conf.Image.Format { // jpeg, webp, ...
		for _, screen := range conf.Image.Screen {
			ic.format = format
			ic.ui = screen.Ui
			ic.density = screen.Density
			ic.width = screen.Width
			if ic.fid, err = imgToFile(&ic); err != nil {
				w.Write(json.Message("ERROR", "Unable to create a file"))
				return
			}
			fid := json.Fid{fmt.Sprintf("%s_%s", screen.Density, screen.Ui), ic.fid}
			result.Image = append(result.Image, fid)
		}
	}
	w.Write(json.Message("OK", &result))
}

func imgToFile(ic *iconf) (string, error) {
	img := resize.Resize(ic.width, 0, ic.image, resize.NearestNeighbor)
	ic.height = uint(img.Bounds().Size().Y)
	ic.fid = fmt.Sprintf("%s-%s-%s-%s-%s-%s-%d-%d", ic.machine, ic.format, ic.density, ic.ui, ic.hash, ic.color, ic.width, ic.height)
	dir := fmt.Sprintf("%s/%s/%s/%s/%s/%s/%s", conf.SushiobrolStore, ic.machine, ic.format, ic.density, ic.ui, ic.hash[0:2], ic.hash[2:4])
	ic.path = fmt.Sprintf("%s/%s", dir, ic.fid)
	out, err := os.Create(ic.path)
	if err != nil {
		log.Println(err)
		return "", err
	}
	defer out.Close()
	if webp.Encode(out, img, &webp.Options{conf.Lossless, conf.Quality}); err != nil {
		fmt.Println("ERROR: Unable to Encode into webp")
		return "", err
	}
	return ic.fid, err
}

func setColor(ic *iconf) {
	img1x1 := resize.Resize(1, 1, ic.image, resize.NearestNeighbor)
	red, green, blue, _ := img1x1.At(0, 0).RGBA()
	ic.color = fmt.Sprintf("%X%X%X", red>>8, green>>8, blue>>8) // removing 1 byte 9A16->9A
}
<|endoftext|>


<commit_before>// Copyright 2015-present, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ccd

import (
	"context"
	"time"

	"github.com/corestoreio/errors"
	"github.com/corestoreio/log"
	"github.com/corestoreio/pkg/sql/ddl"
	"github.com/corestoreio/pkg/sql/dml"
	"github.com/corestoreio/pkg/store/scope"
	"sync"
)

// Options applies options to the DBStorage type.
type Options struct {
	TableName             string
	Log                   log.Logger
	QueryContextTimeout   time.Duration
	IdleAllKeys           time.Duration
	IdleRead              time.Duration
	IdleWrite             time.Duration
	ContextTimeoutAllKeys time.Duration
	ContextTimeoutRead    time.Duration
	ContextTimeoutWrite   time.Duration
}

const (
	stateClosed = iota // must be zero
	stateOpen
	stateInUse
)

// DBStorage connects the MySQL DB with the config.Service type. Implements
// interface config.Storager.
type DBStorage struct {
	cfg Options

	sqlAll   *dml.Select
	sqlRead  *dml.Select
	sqlWrite *dml.Insert

	tickerAll   *time.Ticker
	tickerRead  *time.Ticker
	tickerWrite *time.Ticker

	muAll        sync.Mutex
	stmtAll      *dml.Artisan
	stmtAllState uint8

	muRead        sync.Mutex
	stmtRead      *dml.Artisan
	stmtReadState uint8

	muWrite        sync.Mutex
	stmtWrite      *dml.Artisan
	stmtWriteState uint8
}

// NewDBStorage creates a new database backed storage service. It creates three
// prepared statements which are getting automatically closed after an idle time
// and once used again they get re-prepared.
//
// All has an idle time of 15s. Read an idle time of 10s. Write an idle time of
// 30s. Implements interface config.Storager.
func NewDBStorage(tbls *ddl.Tables, o Options) (*DBStorage, error) {

	tn := o.TableName
	if tn == "" {
		tn = TableNameCoreConfigData
	}

	tbl, err := tbls.Table(tn)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	qryAll := tbl.Select("scope", "scope_id", "path").OrderBy("scope", "scope_id", "path")
	qryAll.Log = o.Log

	qryRead := tbl.Select("value").Where(
		dml.Column("scope").PlaceHolder(),
		dml.Column("scope_id").PlaceHolder(),
		dml.Column("path").PlaceHolder(),
	)
	qryRead.Log = o.Log

	qryWrite := tbl.Insert()
	qryWrite.OnDuplicateKeys = dml.Conditions{
		dml.Column("value"),
	}
	qryWrite.Log = o.Log

	dbs := &DBStorage{
		cfg:      o,
		sqlAll:   qryAll,
		sqlRead:  qryRead,
		sqlWrite: qryWrite,
	}
	if dbs.cfg.IdleAllKeys == 0 {
		dbs.cfg.IdleAllKeys = time.Second * 5 // just a guess
	}
	if dbs.cfg.IdleRead == 0 {
		dbs.cfg.IdleRead = time.Second * 20 // just a guess
	}
	if dbs.cfg.IdleWrite == 0 {
		dbs.cfg.IdleWrite = time.Second * 10 // just a guess
	}
	if dbs.cfg.ContextTimeoutAllKeys == 0 {
		dbs.cfg.ContextTimeoutAllKeys = time.Second * 10 // just a guess
	}
	if dbs.cfg.ContextTimeoutRead == 0 {
		dbs.cfg.ContextTimeoutRead = dbs.cfg.ContextTimeoutAllKeys
	}
	if dbs.cfg.ContextTimeoutWrite == 0 {
		dbs.cfg.ContextTimeoutWrite = dbs.cfg.ContextTimeoutAllKeys
	}
	dbs.tickerAll = time.NewTicker(dbs.cfg.IdleAllKeys)
	dbs.tickerRead = time.NewTicker(dbs.cfg.IdleRead)
	dbs.tickerWrite = time.NewTicker(dbs.cfg.IdleWrite)

	go func() {
		for {
			select {
			case t, ok := <-dbs.tickerAll.C:
				if !ok {
					return
				}
				dbs.muAll.Lock()
				if dbs.stmtAllState == stateOpen {
					if err := dbs.stmtAll.Close(); err != nil && dbs.cfg.Log != nil && dbs.cfg.Log.IsInfo() {
						dbs.cfg.Log.Info("ccd.DBStorage.stmtAll.Close", log.Stringer("ticker", t), log.Err(err))
					}
					dbs.stmtAllState = stateClosed
				}
				dbs.muAll.Unlock()
			case t, ok := <-dbs.tickerRead.C:
				if !ok {
					return
				}
				dbs.muRead.Lock()
				if dbs.stmtReadState == stateOpen {
					if err := dbs.stmtRead.Close(); err != nil && dbs.cfg.Log != nil && dbs.cfg.Log.IsInfo() {
						dbs.cfg.Log.Info("ccd.DBStorage.stmtRead.Close", log.Stringer("ticker", t), log.Err(err))
					}
					dbs.stmtReadState = stateClosed
				}
				dbs.muRead.Unlock()
			case t, ok := <-dbs.tickerWrite.C:
				if !ok {
					return
				}
				dbs.muWrite.Lock()
				if dbs.stmtWriteState == stateOpen {
					if err := dbs.stmtWrite.Close(); err != nil && dbs.cfg.Log != nil && dbs.cfg.Log.IsInfo() {
						dbs.cfg.Log.Info("ccd.DBStorage.stmtWrite.Close", log.Stringer("ticker", t), log.Err(err))
					}
					dbs.stmtWriteState = stateClosed
				}
				dbs.muWrite.Unlock()
			}
		}
	}()

	return dbs, nil
}

// MustNewDBStorage same as NewDBStorage but panics on error. Implements
// interface config.Storager.
func MustNewDBStorage(tbls *ddl.Tables, o Options) *DBStorage {
	s, err := NewDBStorage(tbls, o)
	if err != nil {
		panic(err)
	}
	return s
}

// Close terminates the prepared statements and internal go routines.
func (dbs *DBStorage) Close() error {
	dbs.tickerAll.Stop()
	dbs.tickerRead.Stop()
	dbs.tickerWrite.Stop()
	return nil
}

// Set sets a value with its key. Database errors get logged as Info message.
// Enabled debug level logs the insert ID or rows affected.
func (dbs *DBStorage) Set(scp scope.TypeID, path string, value []byte) error {
	return nil

	// update lastUsed at the end because there might be the slight chance that
	// a statement gets closed despite we're waiting for the result from the
	// server.
	//dbs.Write.StartStmtUse()
	//defer dbs.Write.StopStmtUse()
	//
	//valStr, err := conv.ToStringE(value)
	//if err != nil {
	//	return errors.Wrapf(err, "[ccd] Set.conv.ToStringE. SQL: %q Key: %q Value: %v", dbs.Write.sqlRaw, key, value)
	//}
	//
	//stmt, err := dbs.Write.Stmt(context.TODO())
	//if err != nil {
	//	return errors.Wrapf(err, "[ccd] Set.Write.Stmt. SQL: %q Key: %q", dbs.Write.sqlRaw, key)
	//}
	//
	//pathLeveled, err := key.Level(-1)
	//if err != nil {
	//	return errors.Wrapf(err, "[ccd] Set.key.Level. SQL: %q Key: %q", dbs.Write.sqlRaw, key)
	//}
	//
	//scp, id := key.ScopeID.Unpack()
	//result, err := stmt.Exec(scp.StrType(), id, pathLeveled, valStr, valStr)
	//if err != nil {
	//	return errors.Wrapf(err, "[ccd] Set.stmt.Exec. SQL: %q KeyID: %d Scope: %q Path: %q Value: %q", dbs.Write.sqlRaw, id, scp, pathLeveled, valStr)
	//}
	//if dbs.log.IsDebug() {
	//	li, err1 := result.LastInsertId()
	//	ra, err2 := result.RowsAffected()
	//	dbs.log.Debug(
	//		"config.DBStorage.Set.Write.Result",
	//		log.Int64("lastInsertID", li),
	//		log.ErrWithKey("lastInsertIDErr", err1),
	//		log.Int64("rowsAffected", ra),
	//		log.ErrWithKey("rowsAffectedErr", err2),
	//		log.String("SQL", dbs.Write.sqlRaw),
	//		log.Stringer("key", key),
	//		log.Object("value", value),
	//	)
	//}
	//return nil
}

// Get returns a value from the database by its key. It is guaranteed that the
// type in the empty interface is a string. It returns nil on error but errors
// get logged as info message. Error behaviour: NotFound
func (dbs *DBStorage) Value(scp scope.TypeID, path string) (v []byte, ok bool, err error) {
	return

	// update lastUsed at the end because there might be the slight chance that
	// a statement gets closed despite we're waiting for the result from the
	// server.
	//dbs.Read.StartStmtUse()
	//defer dbs.Read.StopStmtUse()
	//
	//stmt, err := dbs.Read.Stmt(context.TODO())
	//if err != nil {
	//	return nil, errors.Wrapf(err, "[ccd] Get.Read.Stmt. SQL: %q Key: %q", dbs.Read.sqlRaw, key)
	//}
	//
	//pl, err := key.Level(-1)
	//if err != nil {
	//	return nil, errors.Wrapf(err, "[ccd] Get.key.Level. SQL: %q Key: %q", dbs.Read.sqlRaw, key)
	//}
	//
	//var data null.String
	//scp, id := key.ScopeID.Unpack()
	//err = stmt.QueryRow(scp.StrType(), id, pl).Scan(&data)
	//if err != nil {
	//	return nil, errors.Wrapf(err, "[ccd] Get.QueryRow. SQL: %q Key: %q PathLevel: %q", dbs.Read.sqlRaw, key, pl)
	//}
	//if data.Valid {
	//	return data.String, nil
	//}
	//return nil, errKeyNotFound
}

// AllKeys returns all available keys. Database errors get logged as info message.
func (dbs *DBStorage) AllKeys() (scps scope.TypeIDs, paths []string, err error) {
	dbs.muAll.Lock()
	prevState := dbs.stmtAllState
	dbs.stmtAllState = stateInUse
	defer func() {
		dbs.stmtAllState = stateOpen
		dbs.muAll.Unlock()
	}()

	if prevState == stateClosed {
		ctx, cancel := context.WithTimeout(context.Background(), dbs.cfg.ContextTimeoutAllKeys)
		defer cancel()
		var stmt *dml.Stmt
		stmt, err = dbs.sqlAll.Prepare(ctx)
		if err != nil {
			return nil, nil, errors.WithStack(err)
		}
		if dbs.stmtAll == nil {
			dbs.stmtAll = stmt.WithArgs()
		} else {
			dbs.stmtAll.WithStmt(stmt.Stmt)
		}
	}

	scps = make(scope.TypeIDs, 0, 100)
	paths = make([]string, 0, 100)

	dbs.stmtAll.IterateSerial(context.Background(), func(cm *dml.ColumnMap) error {
		var ccd TableCoreConfigData
		if err := ccd.MapColumns(cm); err != nil {
			return errors.Wrapf(err, "[ccd] dbs.stmtAll.IterateSerial at row %d", cm.Count)
		}
		scps = append(scps, scope.MakeTypeID(scope.FromString(ccd.Scope), ccd.ScopeID))
		paths = append(paths, ccd.Path)
		return nil
	})

	return scps, paths, nil
}
<commit_msg>storage/ccd: Initial version of DBStorage.Set<commit_after>// Copyright 2015-present, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ccd

import (
	"context"
	"time"

	"database/sql"
	"github.com/corestoreio/errors"
	"github.com/corestoreio/log"
	"github.com/corestoreio/pkg/sql/ddl"
	"github.com/corestoreio/pkg/sql/dml"
	"github.com/corestoreio/pkg/store/scope"
	"sync"
)

// Options applies options to the DBStorage type.
type Options struct {
	TableName             string
	Log                   log.Logger
	QueryContextTimeout   time.Duration
	IdleAllKeys           time.Duration
	IdleRead              time.Duration
	IdleWrite             time.Duration
	ContextTimeoutAllKeys time.Duration
	ContextTimeoutRead    time.Duration
	ContextTimeoutWrite   time.Duration
}

const (
	stateClosed = iota // must be zero
	stateOpen
	stateInUse
)

// DBStorage connects the MySQL DB with the config.Service type. Implements
// interface config.Storager.
type DBStorage struct {
	cfg Options

	sqlAll   *dml.Select
	sqlRead  *dml.Select
	sqlWrite *dml.Insert

	tickerAll   *time.Ticker
	tickerRead  *time.Ticker
	tickerWrite *time.Ticker

	muAll        sync.Mutex
	stmtAll      *dml.Artisan
	stmtAllState uint8

	muRead        sync.Mutex
	stmtRead      *dml.Artisan
	stmtReadState uint8

	muWrite        sync.Mutex
	stmtWrite      *dml.Artisan
	stmtWriteState uint8
}

// NewDBStorage creates a new database backed storage service. It creates three
// prepared statements which are getting automatically closed after an idle time
// and once used again they get re-prepared.
//
// All has an idle time of 15s. Read an idle time of 10s. Write an idle time of
// 30s. Implements interface config.Storager.
func NewDBStorage(tbls *ddl.Tables, o Options) (*DBStorage, error) {

	tn := o.TableName
	if tn == "" {
		tn = TableNameCoreConfigData
	}

	tbl, err := tbls.Table(tn)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	qryAll := tbl.Select("scope", "scope_id", "path").OrderBy("scope", "scope_id", "path")
	qryAll.Log = o.Log

	qryRead := tbl.Select("value").Where(
		dml.Column("scope").PlaceHolder(),
		dml.Column("scope_id").PlaceHolder(),
		dml.Column("path").PlaceHolder(),
	)
	qryRead.Log = o.Log

	qryWrite := tbl.Insert()
	qryWrite.OnDuplicateKeys = dml.Conditions{
		dml.Column("value"),
	}
	qryWrite.Log = o.Log

	dbs := &DBStorage{
		cfg:      o,
		sqlAll:   qryAll,
		sqlRead:  qryRead,
		sqlWrite: qryWrite,
	}
	if dbs.cfg.IdleAllKeys == 0 {
		dbs.cfg.IdleAllKeys = time.Second * 5 // just a guess
	}
	if dbs.cfg.IdleRead == 0 {
		dbs.cfg.IdleRead = time.Second * 20 // just a guess
	}
	if dbs.cfg.IdleWrite == 0 {
		dbs.cfg.IdleWrite = time.Second * 10 // just a guess
	}
	if dbs.cfg.ContextTimeoutAllKeys == 0 {
		dbs.cfg.ContextTimeoutAllKeys = time.Second * 10 // just a guess
	}
	if dbs.cfg.ContextTimeoutRead == 0 {
		dbs.cfg.ContextTimeoutRead = dbs.cfg.ContextTimeoutAllKeys
	}
	if dbs.cfg.ContextTimeoutWrite == 0 {
		dbs.cfg.ContextTimeoutWrite = dbs.cfg.ContextTimeoutAllKeys
	}
	dbs.tickerAll = time.NewTicker(dbs.cfg.IdleAllKeys)
	dbs.tickerRead = time.NewTicker(dbs.cfg.IdleRead)
	dbs.tickerWrite = time.NewTicker(dbs.cfg.IdleWrite)

	go func() {
		for {
			select {
			case t, ok := <-dbs.tickerAll.C:
				if !ok {
					return
				}
				dbs.muAll.Lock()
				if dbs.stmtAllState == stateOpen {
					if err := dbs.stmtAll.Close(); err != nil && dbs.cfg.Log != nil && dbs.cfg.Log.IsInfo() {
						dbs.cfg.Log.Info("ccd.DBStorage.stmtAll.Close", log.Stringer("ticker", t), log.Err(err))
					}
					dbs.stmtAllState = stateClosed
				}
				dbs.muAll.Unlock()
			case t, ok := <-dbs.tickerRead.C:
				if !ok {
					return
				}
				dbs.muRead.Lock()
				if dbs.stmtReadState == stateOpen {
					if err := dbs.stmtRead.Close(); err != nil && dbs.cfg.Log != nil && dbs.cfg.Log.IsInfo() {
						dbs.cfg.Log.Info("ccd.DBStorage.stmtRead.Close", log.Stringer("ticker", t), log.Err(err))
					}
					dbs.stmtReadState = stateClosed
				}
				dbs.muRead.Unlock()
			case t, ok := <-dbs.tickerWrite.C:
				if !ok {
					return
				}
				dbs.muWrite.Lock()
				if dbs.stmtWriteState == stateOpen {
					if err := dbs.stmtWrite.Close(); err != nil && dbs.cfg.Log != nil && dbs.cfg.Log.IsInfo() {
						dbs.cfg.Log.Info("ccd.DBStorage.stmtWrite.Close", log.Stringer("ticker", t), log.Err(err))
					}
					dbs.stmtWriteState = stateClosed
				}
				dbs.muWrite.Unlock()
			}
		}
	}()

	return dbs, nil
}

// MustNewDBStorage same as NewDBStorage but panics on error. Implements
// interface config.Storager.
func MustNewDBStorage(tbls *ddl.Tables, o Options) *DBStorage {
	s, err := NewDBStorage(tbls, o)
	if err != nil {
		panic(err)
	}
	return s
}

// Close terminates the prepared statements and internal go routines.
func (dbs *DBStorage) Close() error {
	dbs.tickerAll.Stop()
	dbs.tickerRead.Stop()
	dbs.tickerWrite.Stop()
	return nil
}

// Set sets a value with its key. Database errors get logged as Info message.
// Enabled debug level logs the insert ID or rows affected.
func (dbs *DBStorage) Set(scp scope.TypeID, path string, value []byte) (err error) {
	dbs.muAll.Lock()
	prevState := dbs.stmtWriteState
	dbs.stmtWriteState = stateInUse
	defer func() {
		dbs.stmtWriteState = stateOpen
		dbs.muAll.Unlock()
	}()

	if prevState == stateClosed {
		ctx, cancel := context.WithTimeout(context.Background(), dbs.cfg.ContextTimeoutWrite)
		defer cancel()
		var stmt *dml.Stmt
		stmt, err = dbs.sqlWrite.Prepare(ctx)
		if err != nil {
			return errors.WithStack(err)
		}
		if dbs.stmtWrite == nil {
			dbs.stmtWrite = stmt.WithArgs()
		} else {
			dbs.stmtWrite.WithStmt(stmt.Stmt)
		}
	}

	var res sql.Result
	res, err = dbs.stmtWrite.Uint64(scp.ToUint64()).String(path).Bytes(value).ExecContext(context.Background())

	//if dbs.log.IsDebug() {
	//	li, err1 := result.LastInsertId()
	//	ra, err2 := result.RowsAffected()
	//	dbs.log.Debug(
	//		"config.DBStorage.Set.Write.Result",
	//		log.Int64("lastInsertID", li),
	//		log.ErrWithKey("lastInsertIDErr", err1),
	//		log.Int64("rowsAffected", ra),
	//		log.ErrWithKey("rowsAffectedErr", err2),
	//		log.String("SQL", dbs.Write.sqlRaw),
	//		log.Stringer("key", key),
	//		log.Object("value", value),
	//	)
	//}

	return nil
}

// Get returns a value from the database by its key. It is guaranteed that the
// type in the empty interface is a string. It returns nil on error but errors
// get logged as info message. Error behaviour: NotFound
func (dbs *DBStorage) Value(scp scope.TypeID, path string) (v []byte, ok bool, err error) {
	return

	// update lastUsed at the end because there might be the slight chance that
	// a statement gets closed despite we're waiting for the result from the
	// server.
	//dbs.Read.StartStmtUse()
	//defer dbs.Read.StopStmtUse()
	//
	//stmt, err := dbs.Read.Stmt(context.TODO())
	//if err != nil {
	//	return nil, errors.Wrapf(err, "[ccd] Get.Read.Stmt. SQL: %q Key: %q", dbs.Read.sqlRaw, key)
	//}
	//
	//pl, err := key.Level(-1)
	//if err != nil {
	//	return nil, errors.Wrapf(err, "[ccd] Get.key.Level. SQL: %q Key: %q", dbs.Read.sqlRaw, key)
	//}
	//
	//var data null.String
	//scp, id := key.ScopeID.Unpack()
	//err = stmt.QueryRow(scp.StrType(), id, pl).Scan(&data)
	//if err != nil {
	//	return nil, errors.Wrapf(err, "[ccd] Get.QueryRow. SQL: %q Key: %q PathLevel: %q", dbs.Read.sqlRaw, key, pl)
	//}
	//if data.Valid {
	//	return data.String, nil
	//}
	//return nil, errKeyNotFound
}

// AllKeys returns all available keys. Database errors get logged as info message.
func (dbs *DBStorage) AllKeys() (scps scope.TypeIDs, paths []string, err error) {
	dbs.muAll.Lock()
	prevState := dbs.stmtAllState
	dbs.stmtAllState = stateInUse
	defer func() {
		dbs.stmtAllState = stateOpen
		dbs.muAll.Unlock()
	}()

	if prevState == stateClosed {
		ctx, cancel := context.WithTimeout(context.Background(), dbs.cfg.ContextTimeoutAllKeys)
		defer cancel()
		var stmt *dml.Stmt
		stmt, err = dbs.sqlAll.Prepare(ctx)
		if err != nil {
			return nil, nil, errors.WithStack(err)
		}
		if dbs.stmtAll == nil {
			dbs.stmtAll = stmt.WithArgs()
		} else {
			dbs.stmtAll.WithStmt(stmt.Stmt)
		}
	}

	scps = make(scope.TypeIDs, 0, 100)
	paths = make([]string, 0, 100)

	dbs.stmtAll.IterateSerial(context.Background(), func(cm *dml.ColumnMap) error {
		var ccd TableCoreConfigData
		if err := ccd.MapColumns(cm); err != nil {
			return errors.Wrapf(err, "[ccd] dbs.stmtAll.IterateSerial at row %d", cm.Count)
		}
		scps = append(scps, scope.MakeTypeID(scope.FromString(ccd.Scope), ccd.ScopeID))
		paths = append(paths, ccd.Path)
		return nil
	})

	return scps, paths, nil
}
<|endoftext|>


<commit_before>package iiif

// FeaturesMap is a simple map for boolean features, used for comparing
// featuresets and reporting features beyond the reported level
type FeaturesMap map[string]bool

// TileSize represents a supported tile size for a feature set to expose.  This
// data is serialized in an info request and therefore must have JSON tags.
type TileSize struct {
	Width        int   `json:"width"`
	Height       int   `json:"height,omitempty"`
	ScaleFactors []int `json:"scaleFactors"`
}

// FeatureSet represents possible IIIF 2.0 features.  The boolean fields are
// the same as the string to report features, except that the first character
// should be lowercased.
//
// Note that using this in a different server only gets you so far.  As noted
// in the Supported() documentation below, verifying complete support is
// trickier than just checking a URL, and a server that doesn't support
// arbitrary resizing can still advertise specific sizes that will work.
type FeatureSet struct {
	// Region options: note that full isn't specified but must be supported
	RegionByPx   bool
	RegionByPct  bool
	RegionSquare bool

	// Size options: note that full isn't specified but must be supported
	SizeByWhListed    bool
	SizeByW           bool
	SizeByH           bool
	SizeByPct         bool
	SizeByForcedWh    bool
	SizeByWh          bool
	SizeAboveFull     bool
	SizeByConfinedWh  bool
	SizeByDistortedWh bool

	// Rotation and mirroring
	RotationBy90s     bool
	RotationArbitrary bool
	Mirroring         bool

	// "Quality" (color model / color depth)
	Default bool
	Color   bool
	Gray    bool
	Bitonal bool

	// Format
	Jpg  bool
	Png  bool
	Tif  bool
	Gif  bool
	Jp2  bool
	Pdf  bool
	Webp bool

	// HTTP features
	BaseURIRedirect     bool
	Cors                bool
	JsonldMediaType     bool
	ProfileLinkHeader   bool
	CanonicalLinkHeader bool

	// Non-boolean feature support
	TileSizes []TileSize
}

// toMap converts a FeatureSet's boolean support values into a map suitable for
// use in comparison to other feature sets.  The strings used are lowercased so
// they can be used as-is within "formats", "qualities", and/or "supports"
// arrays.
func (fs *FeatureSet) toMap() FeaturesMap {
	return FeaturesMap{
		"regionByPx":          fs.RegionByPx,
		"regionByPct":         fs.RegionByPct,
		"regionSquare":        fs.RegionSquare,
		"sizeByWhListed":      fs.SizeByWhListed,
		"sizeByW":             fs.SizeByW,
		"sizeByH":             fs.SizeByH,
		"sizeByPct":           fs.SizeByPct,
		"sizeByForcedWh":      fs.SizeByForcedWh,
		"sizeByWh":            fs.SizeByWh,
		"sizeByConfinedWh":    fs.SizeByConfinedWh,
		"sizeByDistortedWh":   fs.SizeByDistortedWh,
		"sizeAboveFull":       fs.SizeAboveFull,
		"rotationBy90s":       fs.RotationBy90s,
		"rotationArbitrary":   fs.RotationArbitrary,
		"mirroring":           fs.Mirroring,
		"default":             fs.Default,
		"color":               fs.Color,
		"gray":                fs.Gray,
		"bitonal":             fs.Bitonal,
		"jpg":                 fs.Jpg,
		"png":                 fs.Png,
		"tif":                 fs.Tif,
		"gif":                 fs.Gif,
		"jp2":                 fs.Jp2,
		"pdf":                 fs.Pdf,
		"webp":                fs.Webp,
		"baseUriRedirect":     fs.BaseURIRedirect,
		"cors":                fs.Cors,
		"jsonldMediaType":     fs.JsonldMediaType,
		"profileLinkHeader":   fs.ProfileLinkHeader,
		"canonicalLinkHeader": fs.CanonicalLinkHeader,
	}
}

// FeatureCompare returns which features are in common between two FeatureSets,
// which are exclusive to a, and which are exclusive to b.  The returned maps
// will ONLY contain keys with a value of true, as opposed to the full list of
// features and true/false.  This helps to quickly determine equality, subset
// status, and superset status.
func FeatureCompare(a, b *FeatureSet) (union, onlyA, onlyB FeaturesMap) {
	union = make(FeaturesMap)
	onlyA = make(FeaturesMap)
	onlyB = make(FeaturesMap)

	mapA := a.toMap()
	mapB := b.toMap()

	for feature, supportedA := range mapA {
		supportedB := mapB[feature]
		if supportedA && supportedB {
			union[feature] = true
			continue
		}

		if supportedA {
			onlyA[feature] = true
			continue
		}

		if supportedB {
			onlyB[feature] = true
		}
	}

	return
}

// includes returns whether or not fs includes all features in fsIncluded
func (fs *FeatureSet) includes(fsIncluded *FeatureSet) bool {
	_, _, onlyYours := FeatureCompare(fs, fsIncluded)
	return len(onlyYours) == 0
}
<commit_msg>Fix comment<commit_after>package iiif

// FeaturesMap is a simple map for boolean features, used for comparing
// featuresets and reporting features beyond the reported level
type FeaturesMap map[string]bool

// TileSize represents a supported tile size for a feature set to expose.  This
// data is serialized in an info request and therefore must have JSON tags.
type TileSize struct {
	Width        int   `json:"width"`
	Height       int   `json:"height,omitempty"`
	ScaleFactors []int `json:"scaleFactors"`
}

// FeatureSet represents possible IIIF 2.1 features.  The boolean fields are
// the same as the string to report features, except that the first character
// should be lowercased.
//
// Note that using this in a different server only gets you so far.  As noted
// in the Supported() documentation below, verifying complete support is
// trickier than just checking a URL, and a server that doesn't support
// arbitrary resizing can still advertise specific sizes that will work.
type FeatureSet struct {
	// Region options: note that full isn't specified but must be supported
	RegionByPx   bool
	RegionByPct  bool
	RegionSquare bool

	// Size options: note that full isn't specified but must be supported
	SizeByWhListed    bool
	SizeByW           bool
	SizeByH           bool
	SizeByPct         bool
	SizeByForcedWh    bool
	SizeByWh          bool
	SizeAboveFull     bool
	SizeByConfinedWh  bool
	SizeByDistortedWh bool

	// Rotation and mirroring
	RotationBy90s     bool
	RotationArbitrary bool
	Mirroring         bool

	// "Quality" (color model / color depth)
	Default bool
	Color   bool
	Gray    bool
	Bitonal bool

	// Format
	Jpg  bool
	Png  bool
	Tif  bool
	Gif  bool
	Jp2  bool
	Pdf  bool
	Webp bool

	// HTTP features
	BaseURIRedirect     bool
	Cors                bool
	JsonldMediaType     bool
	ProfileLinkHeader   bool
	CanonicalLinkHeader bool

	// Non-boolean feature support
	TileSizes []TileSize
}

// toMap converts a FeatureSet's boolean support values into a map suitable for
// use in comparison to other feature sets.  The strings used are lowercased so
// they can be used as-is within "formats", "qualities", and/or "supports"
// arrays.
func (fs *FeatureSet) toMap() FeaturesMap {
	return FeaturesMap{
		"regionByPx":          fs.RegionByPx,
		"regionByPct":         fs.RegionByPct,
		"regionSquare":        fs.RegionSquare,
		"sizeByWhListed":      fs.SizeByWhListed,
		"sizeByW":             fs.SizeByW,
		"sizeByH":             fs.SizeByH,
		"sizeByPct":           fs.SizeByPct,
		"sizeByForcedWh":      fs.SizeByForcedWh,
		"sizeByWh":            fs.SizeByWh,
		"sizeByConfinedWh":    fs.SizeByConfinedWh,
		"sizeByDistortedWh":   fs.SizeByDistortedWh,
		"sizeAboveFull":       fs.SizeAboveFull,
		"rotationBy90s":       fs.RotationBy90s,
		"rotationArbitrary":   fs.RotationArbitrary,
		"mirroring":           fs.Mirroring,
		"default":             fs.Default,
		"color":               fs.Color,
		"gray":                fs.Gray,
		"bitonal":             fs.Bitonal,
		"jpg":                 fs.Jpg,
		"png":                 fs.Png,
		"tif":                 fs.Tif,
		"gif":                 fs.Gif,
		"jp2":                 fs.Jp2,
		"pdf":                 fs.Pdf,
		"webp":                fs.Webp,
		"baseUriRedirect":     fs.BaseURIRedirect,
		"cors":                fs.Cors,
		"jsonldMediaType":     fs.JsonldMediaType,
		"profileLinkHeader":   fs.ProfileLinkHeader,
		"canonicalLinkHeader": fs.CanonicalLinkHeader,
	}
}

// FeatureCompare returns which features are in common between two FeatureSets,
// which are exclusive to a, and which are exclusive to b.  The returned maps
// will ONLY contain keys with a value of true, as opposed to the full list of
// features and true/false.  This helps to quickly determine equality, subset
// status, and superset status.
func FeatureCompare(a, b *FeatureSet) (union, onlyA, onlyB FeaturesMap) {
	union = make(FeaturesMap)
	onlyA = make(FeaturesMap)
	onlyB = make(FeaturesMap)

	mapA := a.toMap()
	mapB := b.toMap()

	for feature, supportedA := range mapA {
		supportedB := mapB[feature]
		if supportedA && supportedB {
			union[feature] = true
			continue
		}

		if supportedA {
			onlyA[feature] = true
			continue
		}

		if supportedB {
			onlyB[feature] = true
		}
	}

	return
}

// includes returns whether or not fs includes all features in fsIncluded
func (fs *FeatureSet) includes(fsIncluded *FeatureSet) bool {
	_, _, onlyYours := FeatureCompare(fs, fsIncluded)
	return len(onlyYours) == 0
}
<|endoftext|>


<commit_before>package models

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"time"

	"github.com/technoweenie/grohl"
)

type IMagick struct{}

type processPipelineStep func(workingDirectoryPath string, inputFilePath string, args *ProcessArgs) (outputFilePath string, err error)

var defaultPipeline = []processPipelineStep{
	downloadRemote,
	preProcessImage,
	processImage,
}

// Process a remote asset url using graphicsmagick with the args supplied
// and write the response to w
func (p *IMagick) Process(w http.ResponseWriter, r *http.Request, args *ProcessArgs) (err error) {
	tempDir, err := createTemporaryWorkspace()
	if err != nil {
		return
	}
	// defer os.RemoveAll(tempDir)

	var filePath string

	for _, step := range defaultPipeline {
		filePath, err = step(tempDir, filePath, args)
		if err != nil {
			return
		}
	}

	// serve response
	http.ServeFile(w, r, filePath)
	return
}

func createTemporaryWorkspace() (string, error) {
	return ioutil.TempDir("", "_firesize")
}

func downloadRemote(tempDir string, _ string, args *ProcessArgs) (string, error) {
	url := args.Url
	inFile := filepath.Join(tempDir, "in")

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"download":  url,
		"local":     inFile,
	})

	out, err := os.Create(inFile)
	if err != nil {
		return inFile, err
	}
	defer out.Close()

	resp, err := http.Get(url)
	if err != nil {
		return inFile, err
	}
	defer resp.Body.Close()

	_, err = io.Copy(out, resp.Body)

	return inFile, err
}

func preProcessImage(tempDir string, inFile string, args *ProcessArgs) (string, error) {
	if isAnimatedGif(inFile) {
		args.Format = "gif" // Total hack cos format is incorrectly .png on example
		return coalesceAnimatedGif(tempDir, inFile)
	} else {
		return inFile, nil
	}
}

func processImage(tempDir string, inFile string, args *ProcessArgs) (string, error) {
	outFile := filepath.Join(tempDir, "out")
	cmdArgs, outFileWithFormat := args.CommandArgs(inFile, outFile)

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"args":      cmdArgs,
	})

	executable := "convert"
	cmd := exec.Command(executable, cmdArgs...)
	var outErr bytes.Buffer
	cmd.Stdout, cmd.Stderr = &outErr, &outErr
	err := runWithTimeout(cmd, 60*time.Second)
	if err != nil {
		grohl.Log(grohl.Data{
			"processor": "imagick",
			"failure":   err,
			"args":      cmdArgs,
			"output":    string(outErr.Bytes()),
		})
	}

	return outFileWithFormat, err
}

func isAnimatedGif(inFile string) bool {
	// identify -format %n updates-product-click.gif # => 105
	cmd := exec.Command("identify", "-format", "%n", inFile)
	var out bytes.Buffer
	cmd.Stdout = &out
	err := runWithTimeout(cmd, 10*time.Second)
	if err == nil {
		numFrames, err := strconv.Atoi(string(out.Bytes()))
		if err == nil {
			grohl.Log(grohl.Data{
				"processor":  "imagick",
				"num-frames": numFrames,
			})
			return numFrames > 1
		}
	}
	// if anything fucks out assume not animated
	return false
}

func coalesceAnimatedGif(tempDir string, inFile string) (string, error) {
	outFile := filepath.Join(tempDir, "temp")

	// convert do.gif -coalesce temporary.gif
	cmd := exec.Command("convert", inFile, "-coalesce", outFile)
	_ = runWithTimeout(cmd, 60*time.Second)

	return outFile, nil
}

func runWithTimeout(cmd *exec.Cmd, timeout time.Duration) error {
	// Start the process
	err := cmd.Start()
	if err != nil {
		return err
	}

	// Kill the process if it doesn't exit in time
	defer time.AfterFunc(timeout, func() {
		fmt.Println("command timed out")
		cmd.Process.Kill()
	}).Stop()

	// Wait for the process to finish
	return cmd.Wait()
}
<commit_msg>Add extra logging for identify output.<commit_after>package models

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"time"

	"github.com/technoweenie/grohl"
)

type IMagick struct{}

type processPipelineStep func(workingDirectoryPath string, inputFilePath string, args *ProcessArgs) (outputFilePath string, err error)

var defaultPipeline = []processPipelineStep{
	downloadRemote,
	preProcessImage,
	processImage,
}

// Process a remote asset url using graphicsmagick with the args supplied
// and write the response to w
func (p *IMagick) Process(w http.ResponseWriter, r *http.Request, args *ProcessArgs) (err error) {
	tempDir, err := createTemporaryWorkspace()
	if err != nil {
		return
	}
	// defer os.RemoveAll(tempDir)

	var filePath string

	for _, step := range defaultPipeline {
		filePath, err = step(tempDir, filePath, args)
		if err != nil {
			return
		}
	}

	// serve response
	http.ServeFile(w, r, filePath)
	return
}

func createTemporaryWorkspace() (string, error) {
	return ioutil.TempDir("", "_firesize")
}

func downloadRemote(tempDir string, _ string, args *ProcessArgs) (string, error) {
	url := args.Url
	inFile := filepath.Join(tempDir, "in")

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"download":  url,
		"local":     inFile,
	})

	out, err := os.Create(inFile)
	if err != nil {
		return inFile, err
	}
	defer out.Close()

	resp, err := http.Get(url)
	if err != nil {
		return inFile, err
	}
	defer resp.Body.Close()

	_, err = io.Copy(out, resp.Body)

	return inFile, err
}

func preProcessImage(tempDir string, inFile string, args *ProcessArgs) (string, error) {
	if isAnimatedGif(inFile) {
		args.Format = "gif" // Total hack cos format is incorrectly .png on example
		return coalesceAnimatedGif(tempDir, inFile)
	} else {
		return inFile, nil
	}
}

func processImage(tempDir string, inFile string, args *ProcessArgs) (string, error) {
	outFile := filepath.Join(tempDir, "out")
	cmdArgs, outFileWithFormat := args.CommandArgs(inFile, outFile)

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"args":      cmdArgs,
	})

	executable := "convert"
	cmd := exec.Command(executable, cmdArgs...)
	var outErr bytes.Buffer
	cmd.Stdout, cmd.Stderr = &outErr, &outErr
	err := runWithTimeout(cmd, 60*time.Second)
	if err != nil {
		grohl.Log(grohl.Data{
			"processor": "imagick",
			"failure":   err,
			"args":      cmdArgs,
			"output":    string(outErr.Bytes()),
		})
	}

	return outFileWithFormat, err
}

func isAnimatedGif(inFile string) bool {
	// identify -format %n updates-product-click.gif # => 105
	cmd := exec.Command("identify", "-format", "%n", inFile)
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := runWithTimeout(cmd, 10*time.Second)
	if err != nil {
		output := string(stderr.Bytes())
		grohl.Log(grohl.Data{
			"identify-error": err,
			"raw-stderr":     output,
		})
	} else {
		output := string(stdout.Bytes())
		numFrames, err := strconv.Atoi(output)
		if err != nil {
			grohl.Log(grohl.Data{
				"identify-error": "non numeric identify output",
				"raw-stdout":     output,
			})
		} else {
			grohl.Log(grohl.Data{
				"processor":  "imagick",
				"num-frames": numFrames,
			})
			return numFrames > 1
		}
	}
	// if anything fucks out assume not animated
	return false
}

func coalesceAnimatedGif(tempDir string, inFile string) (string, error) {
	outFile := filepath.Join(tempDir, "temp")

	// convert do.gif -coalesce temporary.gif
	cmd := exec.Command("convert", inFile, "-coalesce", outFile)
	_ = runWithTimeout(cmd, 60*time.Second)

	return outFile, nil
}

func runWithTimeout(cmd *exec.Cmd, timeout time.Duration) error {
	// Start the process
	err := cmd.Start()
	if err != nil {
		return err
	}

	// Kill the process if it doesn't exit in time
	defer time.AfterFunc(timeout, func() {
		fmt.Println("command timed out")
		cmd.Process.Kill()
	}).Stop()

	// Wait for the process to finish
	return cmd.Wait()
}
<|endoftext|>