<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package proc implements a partial in-memory file system for profs.
package proc

import (
	"fmt"
	"io"
	"sort"
	"strconv"

	"gvisor.googlesource.com/gvisor/pkg/sentry/context"
	"gvisor.googlesource.com/gvisor/pkg/sentry/fs"
	"gvisor.googlesource.com/gvisor/pkg/sentry/fs/proc/device"
	"gvisor.googlesource.com/gvisor/pkg/sentry/fs/proc/seqfile"
	"gvisor.googlesource.com/gvisor/pkg/sentry/fs/ramfs"
	"gvisor.googlesource.com/gvisor/pkg/sentry/kernel"
	"gvisor.googlesource.com/gvisor/pkg/sentry/socket/rpcinet"
	"gvisor.googlesource.com/gvisor/pkg/sentry/usermem"
	"gvisor.googlesource.com/gvisor/pkg/syserror"
)

// proc is a root proc node.
//
// +stateify savable
type proc struct {
	ramfs.Dir

	// k is the Kernel containing this proc node.
	k *kernel.Kernel

	// pidns is the PID namespace of the task that mounted the proc filesystem
	// that this node represents.
	pidns *kernel.PIDNamespace
}

// stubProcFSFile is a file type that can be used to return file contents
// which are constant. This file is not writable and will always have mode
// 0444.
//
// +stateify savable
type stubProcFSFile struct {
	ramfs.Entry

	// contents are the immutable file contents that will always be returned.
	contents []byte
}

// DeprecatedPreadv implements fs.InodeOperations.DeprecatedPreadv.
func (s *stubProcFSFile) DeprecatedPreadv(ctx context.Context, dst usermem.IOSequence, offset int64) (int64, error) {
	if offset < 0 {
		return 0, syserror.EINVAL
	}

	if offset >= int64(len(s.contents)) {
		return 0, io.EOF
	}

	n, err := dst.CopyOut(ctx, s.contents[offset:])
	return int64(n), err
}

// New returns the root node of a partial simple procfs.
func New(ctx context.Context, msrc *fs.MountSource) (*fs.Inode, error) {
	k := kernel.KernelFromContext(ctx)
	if k == nil {
		return nil, fmt.Errorf("procfs requires a kernel")
	}
	pidns := kernel.PIDNamespaceFromContext(ctx)
	if pidns == nil {
		return nil, fmt.Errorf("procfs requires a PID namespace")
	}

	p := &proc{k: k, pidns: pidns}
	p.InitDir(ctx, map[string]*fs.Inode{
		// Note that these are just the static members. There are
		// dynamic members populated in Readdir and Lookup below.
		"filesystems": seqfile.NewSeqFileInode(ctx, &filesystemsData{}, msrc),
		"loadavg":     seqfile.NewSeqFileInode(ctx, &loadavgData{}, msrc),
		"meminfo":     seqfile.NewSeqFileInode(ctx, &meminfoData{k}, msrc),
		"mounts":      newMountsSymlink(ctx, msrc),
		"stat":        seqfile.NewSeqFileInode(ctx, &statData{k}, msrc),
		"version":     seqfile.NewSeqFileInode(ctx, &versionData{k}, msrc),
	}, fs.RootOwner, fs.FilePermsFromMode(0555))

	p.AddChild(ctx, "cpuinfo", p.newCPUInfo(ctx, msrc))
	p.AddChild(ctx, "uptime", p.newUptime(ctx, msrc))

	return newFile(p, msrc, fs.SpecialDirectory, nil), nil
}

// self is a magical link.
type self struct {
	ramfs.Symlink

	pidns *kernel.PIDNamespace
}

// newSelf returns a new "self" node.
func (p *proc) newSelf(ctx context.Context, msrc *fs.MountSource) *fs.Inode {
	s := &self{pidns: p.pidns}
	s.InitSymlink(ctx, fs.RootOwner, "")
	return newFile(s, msrc, fs.Symlink, nil)
}

// newThreadSelf returns a new "threadSelf" node.
func (p *proc) newThreadSelf(ctx context.Context, msrc *fs.MountSource) *fs.Inode {
	s := &threadSelf{pidns: p.pidns}
	s.InitSymlink(ctx, fs.RootOwner, "")
	return newFile(s, msrc, fs.Symlink, nil)
}

// newStubProcFsFile returns a procfs file with constant contents.
func (p *proc) newStubProcFSFile(ctx context.Context, msrc *fs.MountSource, c []byte) *fs.Inode {
	u := &stubProcFSFile{
		contents: c,
	}
	u.InitEntry(ctx, fs.RootOwner, fs.FilePermsFromMode(0444))
	return newFile(u, msrc, fs.SpecialFile, nil)
}

// Readlink implements fs.InodeOperations.Readlink.
func (s *self) Readlink(ctx context.Context, inode *fs.Inode) (string, error) {
	if t := kernel.TaskFromContext(ctx); t != nil {
		tgid := s.pidns.IDOfThreadGroup(t.ThreadGroup())
		if tgid == 0 {
			return "", ramfs.ErrNotFound
		}
		return strconv.FormatUint(uint64(tgid), 10), nil
	}

	// Who is reading this link?
	return "", ramfs.ErrInvalidOp
}

// threadSelf is more magical than "self" link.
type threadSelf struct {
	ramfs.Symlink

	pidns *kernel.PIDNamespace
}

// Readlink implements fs.InodeOperations.Readlink.
func (s *threadSelf) Readlink(ctx context.Context, inode *fs.Inode) (string, error) {
	if t := kernel.TaskFromContext(ctx); t != nil {
		tgid := s.pidns.IDOfThreadGroup(t.ThreadGroup())
		tid := s.pidns.IDOfTask(t)
		if tid == 0 || tgid == 0 {
			return "", ramfs.ErrNotFound
		}
		return fmt.Sprintf("%d/task/%d", tgid, tid), nil
	}

	// Who is reading this link?
	return "", ramfs.ErrInvalidOp
}

// Lookup loads an Inode at name into a Dirent.
func (p *proc) Lookup(ctx context.Context, dir *fs.Inode, name string) (*fs.Dirent, error) {
	// Is it one of the static ones?
	dirent, walkErr := p.Dir.Lookup(ctx, dir, name)
	if walkErr == nil {
		return dirent, nil
	}

	// Is it a dynamic element?
	nfs := map[string]func() *fs.Inode{
		"net": func() *fs.Inode {
			// If we're using rpcinet we will let it manage /proc/net.
			if _, ok := p.k.NetworkStack().(*rpcinet.Stack); ok {
				return newRPCInetProcNet(ctx, dir.MountSource)
			}
			return p.newNetDir(ctx, dir.MountSource)
		},
		"self":        func() *fs.Inode { return p.newSelf(ctx, dir.MountSource) },
		"sys":         func() *fs.Inode { return p.newSysDir(ctx, dir.MountSource) },
		"thread-self": func() *fs.Inode { return p.newThreadSelf(ctx, dir.MountSource) },
	}
	if nf, ok := nfs[name]; ok {
		return fs.NewDirent(nf(), name), nil
	}

	// Try to lookup a corresponding task.
	tid, err := strconv.ParseUint(name, 10, 64)
	if err != nil {
		// Ignore the parse error and return the original.
		return nil, walkErr
	}

	// Grab the other task.
	otherTask := p.pidns.TaskWithID(kernel.ThreadID(tid))
	if otherTask == nil {
		// Per above.
		return nil, walkErr
	}

	// Wrap it in a taskDir.
	td := newTaskDir(otherTask, dir.MountSource, p.pidns, true)
	return fs.NewDirent(td, name), nil
}

// Readdir synthesizes proc contents.
func (p *proc) DeprecatedReaddir(ctx context.Context, dirCtx *fs.DirCtx, offset int) (int, error) {
	// Serialize normal contents.
	_, err := p.Dir.DeprecatedReaddir(ctx, dirCtx, offset)
	if err != nil {
		return offset, err
	}

	m := make(map[string]fs.DentAttr)
	var names []string

	// Add special files.
	m["sys"] = fs.GenericDentAttr(fs.SpecialFile, device.ProcDevice)
	names = append(names, "sys")

	// Collect tasks.
	// Per linux we only include it in directory listings if it's the leader.
	// But for whatever crazy reason, you can still walk to the given node.
	for _, tg := range p.pidns.ThreadGroups() {
		if leader := tg.Leader(); leader != nil {
			name := strconv.FormatUint(uint64(tg.ID()), 10)
			m[name] = fs.GenericDentAttr(fs.SpecialDirectory, device.ProcDevice)
			names = append(names, name)
		}
	}

	if offset >= len(m) {
		return offset, nil
	}
	sort.Strings(names)
	names = names[offset:]
	for _, name := range names {
		if err := dirCtx.DirEmit(name, m[name]); err != nil {
			return offset, err
		}
		offset++
	}
	return offset, err
}

// newMountsSymlink returns a symlink to "self/mounts"
func newMountsSymlink(ctx context.Context, msrc *fs.MountSource) *fs.Inode {
	s := &ramfs.Symlink{}
	s.InitSymlink(ctx, fs.RootOwner, "self/mounts")
	return newFile(s, msrc, fs.Symlink, nil)
}
<commit_msg>Add period to comment<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package proc implements a partial in-memory file system for profs.
package proc

import (
	"fmt"
	"io"
	"sort"
	"strconv"

	"gvisor.googlesource.com/gvisor/pkg/sentry/context"
	"gvisor.googlesource.com/gvisor/pkg/sentry/fs"
	"gvisor.googlesource.com/gvisor/pkg/sentry/fs/proc/device"
	"gvisor.googlesource.com/gvisor/pkg/sentry/fs/proc/seqfile"
	"gvisor.googlesource.com/gvisor/pkg/sentry/fs/ramfs"
	"gvisor.googlesource.com/gvisor/pkg/sentry/kernel"
	"gvisor.googlesource.com/gvisor/pkg/sentry/socket/rpcinet"
	"gvisor.googlesource.com/gvisor/pkg/sentry/usermem"
	"gvisor.googlesource.com/gvisor/pkg/syserror"
)

// proc is a root proc node.
//
// +stateify savable
type proc struct {
	ramfs.Dir

	// k is the Kernel containing this proc node.
	k *kernel.Kernel

	// pidns is the PID namespace of the task that mounted the proc filesystem
	// that this node represents.
	pidns *kernel.PIDNamespace
}

// stubProcFSFile is a file type that can be used to return file contents
// which are constant. This file is not writable and will always have mode
// 0444.
//
// +stateify savable
type stubProcFSFile struct {
	ramfs.Entry

	// contents are the immutable file contents that will always be returned.
	contents []byte
}

// DeprecatedPreadv implements fs.InodeOperations.DeprecatedPreadv.
func (s *stubProcFSFile) DeprecatedPreadv(ctx context.Context, dst usermem.IOSequence, offset int64) (int64, error) {
	if offset < 0 {
		return 0, syserror.EINVAL
	}

	if offset >= int64(len(s.contents)) {
		return 0, io.EOF
	}

	n, err := dst.CopyOut(ctx, s.contents[offset:])
	return int64(n), err
}

// New returns the root node of a partial simple procfs.
func New(ctx context.Context, msrc *fs.MountSource) (*fs.Inode, error) {
	k := kernel.KernelFromContext(ctx)
	if k == nil {
		return nil, fmt.Errorf("procfs requires a kernel")
	}
	pidns := kernel.PIDNamespaceFromContext(ctx)
	if pidns == nil {
		return nil, fmt.Errorf("procfs requires a PID namespace")
	}

	p := &proc{k: k, pidns: pidns}
	p.InitDir(ctx, map[string]*fs.Inode{
		// Note that these are just the static members. There are
		// dynamic members populated in Readdir and Lookup below.
		"filesystems": seqfile.NewSeqFileInode(ctx, &filesystemsData{}, msrc),
		"loadavg":     seqfile.NewSeqFileInode(ctx, &loadavgData{}, msrc),
		"meminfo":     seqfile.NewSeqFileInode(ctx, &meminfoData{k}, msrc),
		"mounts":      newMountsSymlink(ctx, msrc),
		"stat":        seqfile.NewSeqFileInode(ctx, &statData{k}, msrc),
		"version":     seqfile.NewSeqFileInode(ctx, &versionData{k}, msrc),
	}, fs.RootOwner, fs.FilePermsFromMode(0555))

	p.AddChild(ctx, "cpuinfo", p.newCPUInfo(ctx, msrc))
	p.AddChild(ctx, "uptime", p.newUptime(ctx, msrc))

	return newFile(p, msrc, fs.SpecialDirectory, nil), nil
}

// self is a magical link.
type self struct {
	ramfs.Symlink

	pidns *kernel.PIDNamespace
}

// newSelf returns a new "self" node.
func (p *proc) newSelf(ctx context.Context, msrc *fs.MountSource) *fs.Inode {
	s := &self{pidns: p.pidns}
	s.InitSymlink(ctx, fs.RootOwner, "")
	return newFile(s, msrc, fs.Symlink, nil)
}

// newThreadSelf returns a new "threadSelf" node.
func (p *proc) newThreadSelf(ctx context.Context, msrc *fs.MountSource) *fs.Inode {
	s := &threadSelf{pidns: p.pidns}
	s.InitSymlink(ctx, fs.RootOwner, "")
	return newFile(s, msrc, fs.Symlink, nil)
}

// newStubProcFsFile returns a procfs file with constant contents.
func (p *proc) newStubProcFSFile(ctx context.Context, msrc *fs.MountSource, c []byte) *fs.Inode {
	u := &stubProcFSFile{
		contents: c,
	}
	u.InitEntry(ctx, fs.RootOwner, fs.FilePermsFromMode(0444))
	return newFile(u, msrc, fs.SpecialFile, nil)
}

// Readlink implements fs.InodeOperations.Readlink.
func (s *self) Readlink(ctx context.Context, inode *fs.Inode) (string, error) {
	if t := kernel.TaskFromContext(ctx); t != nil {
		tgid := s.pidns.IDOfThreadGroup(t.ThreadGroup())
		if tgid == 0 {
			return "", ramfs.ErrNotFound
		}
		return strconv.FormatUint(uint64(tgid), 10), nil
	}

	// Who is reading this link?
	return "", ramfs.ErrInvalidOp
}

// threadSelf is more magical than "self" link.
type threadSelf struct {
	ramfs.Symlink

	pidns *kernel.PIDNamespace
}

// Readlink implements fs.InodeOperations.Readlink.
func (s *threadSelf) Readlink(ctx context.Context, inode *fs.Inode) (string, error) {
	if t := kernel.TaskFromContext(ctx); t != nil {
		tgid := s.pidns.IDOfThreadGroup(t.ThreadGroup())
		tid := s.pidns.IDOfTask(t)
		if tid == 0 || tgid == 0 {
			return "", ramfs.ErrNotFound
		}
		return fmt.Sprintf("%d/task/%d", tgid, tid), nil
	}

	// Who is reading this link?
	return "", ramfs.ErrInvalidOp
}

// Lookup loads an Inode at name into a Dirent.
func (p *proc) Lookup(ctx context.Context, dir *fs.Inode, name string) (*fs.Dirent, error) {
	// Is it one of the static ones?
	dirent, walkErr := p.Dir.Lookup(ctx, dir, name)
	if walkErr == nil {
		return dirent, nil
	}

	// Is it a dynamic element?
	nfs := map[string]func() *fs.Inode{
		"net": func() *fs.Inode {
			// If we're using rpcinet we will let it manage /proc/net.
			if _, ok := p.k.NetworkStack().(*rpcinet.Stack); ok {
				return newRPCInetProcNet(ctx, dir.MountSource)
			}
			return p.newNetDir(ctx, dir.MountSource)
		},
		"self":        func() *fs.Inode { return p.newSelf(ctx, dir.MountSource) },
		"sys":         func() *fs.Inode { return p.newSysDir(ctx, dir.MountSource) },
		"thread-self": func() *fs.Inode { return p.newThreadSelf(ctx, dir.MountSource) },
	}
	if nf, ok := nfs[name]; ok {
		return fs.NewDirent(nf(), name), nil
	}

	// Try to lookup a corresponding task.
	tid, err := strconv.ParseUint(name, 10, 64)
	if err != nil {
		// Ignore the parse error and return the original.
		return nil, walkErr
	}

	// Grab the other task.
	otherTask := p.pidns.TaskWithID(kernel.ThreadID(tid))
	if otherTask == nil {
		// Per above.
		return nil, walkErr
	}

	// Wrap it in a taskDir.
	td := newTaskDir(otherTask, dir.MountSource, p.pidns, true)
	return fs.NewDirent(td, name), nil
}

// Readdir synthesizes proc contents.
func (p *proc) DeprecatedReaddir(ctx context.Context, dirCtx *fs.DirCtx, offset int) (int, error) {
	// Serialize normal contents.
	_, err := p.Dir.DeprecatedReaddir(ctx, dirCtx, offset)
	if err != nil {
		return offset, err
	}

	m := make(map[string]fs.DentAttr)
	var names []string

	// Add special files.
	m["sys"] = fs.GenericDentAttr(fs.SpecialFile, device.ProcDevice)
	names = append(names, "sys")

	// Collect tasks.
	// Per linux we only include it in directory listings if it's the leader.
	// But for whatever crazy reason, you can still walk to the given node.
	for _, tg := range p.pidns.ThreadGroups() {
		if leader := tg.Leader(); leader != nil {
			name := strconv.FormatUint(uint64(tg.ID()), 10)
			m[name] = fs.GenericDentAttr(fs.SpecialDirectory, device.ProcDevice)
			names = append(names, name)
		}
	}

	if offset >= len(m) {
		return offset, nil
	}
	sort.Strings(names)
	names = names[offset:]
	for _, name := range names {
		if err := dirCtx.DirEmit(name, m[name]); err != nil {
			return offset, err
		}
		offset++
	}
	return offset, err
}

// newMountsSymlink returns a symlink to "self/mounts".
func newMountsSymlink(ctx context.Context, msrc *fs.MountSource) *fs.Inode {
	s := &ramfs.Symlink{}
	s.InitSymlink(ctx, fs.RootOwner, "self/mounts")
	return newFile(s, msrc, fs.Symlink, nil)
}
<|endoftext|>


<commit_before>/*
 * Mini Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package donut

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"io"
)

type Header struct{}
type Donut struct {
	file io.Writer
}

func (donut Donut) Write(header Header, object io.Reader) error {
	var newObjectBuffer bytes.Buffer
	var headerBuffer bytes.Buffer

	// create gob header
	headerEncoder := gob.NewEncoder(&headerBuffer)
	err := headerEncoder.Encode(header)
	if err != nil {
		return err
	}

	// write header length
	var headerLengthBuffer bytes.Buffer
	headerLength := headerBuffer.Len()
	err = binary.Write(&headerLengthBuffer, binary.LittleEndian, headerLength)
	if err != nil {
		return err
	}

	// write header length
	io.Copy(&newObjectBuffer, &headerLengthBuffer)

	// write header
	io.Copy(&newObjectBuffer, &headerBuffer)

	// write header marker

	// TODO

	// write data
	_, err = io.Copy(&newObjectBuffer, object)
	if err != nil {
		return err
	}

	// write footer
	// TODO

	// write footer marker
	// TODO

	// write new object
	_, err = io.Copy(donut.file, &newObjectBuffer)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Adding header version<commit_after>/*
 * Mini Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package donut

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"io"
)

type Header struct{}
type Donut struct {
	file io.Writer
}

func (donut Donut) Write(header Header, object io.Reader) error {
	var newObjectBuffer bytes.Buffer
	var headerBuffer bytes.Buffer

	// create gob header
	headerEncoder := gob.NewEncoder(&headerBuffer)
	err := headerEncoder.Encode(header)
	if err != nil {
		return err
	}

	// prefix consists of a version number and a length
	var headerPrefixBuffer bytes.Buffer
	// write version
	var version int
	version = 1
	err = binary.Write(&headerPrefixBuffer, binary.LittleEndian, version)
	if err != nil {
		return err
	}

	// write length
	var headerLength int
	headerLength = headerBuffer.Len()
	err = binary.Write(&headerPrefixBuffer, binary.LittleEndian, headerLength)
	if err != nil {
		return err
	}

	// write header prefix
	io.Copy(&newObjectBuffer, &headerPrefixBuffer)

	// write header
	io.Copy(&newObjectBuffer, &headerBuffer)

	// write header marker

	// TODO

	// write data
	_, err = io.Copy(&newObjectBuffer, object)
	if err != nil {
		return err
	}

	// write footer
	// TODO

	// write footer marker
	// TODO

	// write new object
	_, err = io.Copy(donut.file, &newObjectBuffer)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Package heroku is a client interface to the Heroku API.
package heroku

import (
	"bytes"
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"runtime"
	"strings"
)

const (
	Version          = "0.1"
	DefaultUserAgent = "heroku-go " + Version + " " + runtime.GOOS + " " + runtime.GOARCH
)

// A Client is a Heroku API client. Its zero value is a usable client that uses
// default settings for the Heroku API. The Client has an internal HTTP client
// (HTTP) which defaults to http.DefaultClient.
//
// As with all http.Clients, this Client's Transport has internal state (cached
// HTTP connections), so Clients should be reused instead of created as needed.
// Clients are safe for use by multiple goroutines.
type Client struct {
	// HTTP is the Client's internal http.Client, handling HTTP requests to the
	// Heroku API.
	HTTP *http.Client

	// The URL of the Heroku API to communicate with. Defaults to
	// "https://api.heroku.com".
	URL string

	// Username is the HTTP basic auth username for API calls made by this Client.
	Username string

	// Password is the HTTP basic auth password for API calls made by this Client.
	Password string

	// UserAgent to be provided in API requests. Set to DefaultUserAgent if not
	// specified.
	UserAgent string
}

func (c *Client) Get(v interface{}, path string) error {
	return c.APIReq(v, "GET", path, nil)
}

func (c *Client) Patch(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "PATCH", path, body)
}

func (c *Client) Post(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "POST", path, body)
}

func (c *Client) Put(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "PUT", path, body)
}

func (c *Client) Delete(path string) error {
	return c.APIReq(nil, "DELETE", path, nil)
}

// Generates an HTTP request for the Heroku API, but does not
// perform the request. The request's Accept header field will be
// set to:
//
//   Accept: application/vnd.heroku+json; version=3
//
// The Request-Id header will be set to a random UUID. The User-Agent header
// will be set to the Client's UserAgent, or DefaultUserAgent if UserAgent is
// not set.
//
// The type of body determines how to encode the request:
//
//   nil         no body
//   io.Reader   body is sent verbatim
//   else        body is encoded as application/json
func (c *Client) NewRequest(method, path string, body interface{}) (*http.Request, error) {
	var ctype string
	var rbody io.Reader

	switch t := body.(type) {
	case nil:
	case string:
		rbody = bytes.NewBufferString(t)
	case io.Reader:
		rbody = t
	default:
		j, err := json.Marshal(body)
		if err != nil {
			log.Fatal(err)
		}
		rbody = bytes.NewReader(j)
		ctype = "application/json"
	}
	apiURL := strings.TrimRight(c.URL, "/")
	if apiURL == "" {
		apiURL = "https://api.heroku.com"
	}
	req, err := http.NewRequest(method, apiURL+path, rbody)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/vnd.heroku+json; version=3")
	req.Header.Set("Request-Id", uuid.New())
	useragent := c.UserAgent
	if useragent == "" {
		useragent = DefaultUserAgent
	}
	req.Header.Set("User-Agent", useragent)
	if ctype != "" {
		req.Header.Set("Content-Type", ctype)
	}
	req.SetBasicAuth(c.Username, c.Password)
	for _, h := range strings.Split(os.Getenv("HKHEADER"), "\n") {
		if i := strings.Index(h, ":"); i >= 0 {
			req.Header.Set(
				strings.TrimSpace(h[:i]),
				strings.TrimSpace(h[i+1:]),
			)
		}
	}
	return req, nil
}

// Sends a Heroku API request and decodes the response into v. As
// described in NewRequest(), the type of body determines how to
// encode the request body. As described in DoReq(), the type of
// v determines how to handle the response body.
func (c *Client) APIReq(v interface{}, meth, path string, body interface{}) error {
	req, err := c.NewRequest(meth, path, body)
	if err != nil {
		return err
	}
	return c.DoReq(req, v)
}

// Submits an HTTP request, checks its response, and deserializes
// the response into v. The type of v determines how to handle
// the response body:
//
//   nil        body is discarded
//   io.Writer  body is copied directly into v
//   else       body is decoded into v as json
//
func (c *Client) DoReq(req *http.Request, v interface{}) error {
	debug := os.Getenv("HKDEBUG") != ""
	if debug {
		dump, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			log.Println(err)
		} else {
			os.Stderr.Write(dump)
			os.Stderr.Write([]byte{'\n', '\n'})
		}
	}

	httpClient := c.HTTP
	if httpClient == nil {
		httpClient = http.DefaultClient
	}

	res, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if debug {
		dump, err := httputil.DumpResponse(res, true)
		if err != nil {
			log.Println(err)
		} else {
			os.Stderr.Write(dump)
			os.Stderr.Write([]byte{'\n'})
		}
	}
	if err = checkResp(res); err != nil {
		return err
	}
	switch t := v.(type) {
	case nil:
	case io.Writer:
		_, err = io.Copy(t, res.Body)
	default:
		err = json.NewDecoder(res.Body).Decode(v)
	}
	return err
}

func checkResp(res *http.Response) error {
	if res.StatusCode == 401 {
		return errors.New("Unauthorized")
	}
	if res.StatusCode == 403 {
		return errors.New("Unauthorized")
	}
	if res.StatusCode/100 != 2 { // 200, 201, 202, etc
		return errors.New("Unexpected error: " + res.Status)
	}
	if msg := res.Header.Get("X-Heroku-Warning"); msg != "" {
		fmt.Fprintln(os.Stderr, strings.TrimSpace(msg))
	}
	return nil
}
<commit_msg>ListRange type to simplify Range headers<commit_after>// Package heroku is a client interface to the Heroku API.
package heroku

import (
	"bytes"
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"runtime"
	"strings"
)

const (
	Version          = "0.1"
	DefaultUserAgent = "heroku-go " + Version + " " + runtime.GOOS + " " + runtime.GOARCH
)

// A Client is a Heroku API client. Its zero value is a usable client that uses
// default settings for the Heroku API. The Client has an internal HTTP client
// (HTTP) which defaults to http.DefaultClient.
//
// As with all http.Clients, this Client's Transport has internal state (cached
// HTTP connections), so Clients should be reused instead of created as needed.
// Clients are safe for use by multiple goroutines.
type Client struct {
	// HTTP is the Client's internal http.Client, handling HTTP requests to the
	// Heroku API.
	HTTP *http.Client

	// The URL of the Heroku API to communicate with. Defaults to
	// "https://api.heroku.com".
	URL string

	// Username is the HTTP basic auth username for API calls made by this Client.
	Username string

	// Password is the HTTP basic auth password for API calls made by this Client.
	Password string

	// UserAgent to be provided in API requests. Set to DefaultUserAgent if not
	// specified.
	UserAgent string
}

func (c *Client) Get(v interface{}, path string) error {
	return c.APIReq(v, "GET", path, nil)
}

func (c *Client) Patch(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "PATCH", path, body)
}

func (c *Client) Post(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "POST", path, body)
}

func (c *Client) Put(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "PUT", path, body)
}

func (c *Client) Delete(path string) error {
	return c.APIReq(nil, "DELETE", path, nil)
}

// Generates an HTTP request for the Heroku API, but does not
// perform the request. The request's Accept header field will be
// set to:
//
//   Accept: application/vnd.heroku+json; version=3
//
// The Request-Id header will be set to a random UUID. The User-Agent header
// will be set to the Client's UserAgent, or DefaultUserAgent if UserAgent is
// not set.
//
// The type of body determines how to encode the request:
//
//   nil         no body
//   io.Reader   body is sent verbatim
//   else        body is encoded as application/json
func (c *Client) NewRequest(method, path string, body interface{}) (*http.Request, error) {
	var ctype string
	var rbody io.Reader

	switch t := body.(type) {
	case nil:
	case string:
		rbody = bytes.NewBufferString(t)
	case io.Reader:
		rbody = t
	default:
		j, err := json.Marshal(body)
		if err != nil {
			log.Fatal(err)
		}
		rbody = bytes.NewReader(j)
		ctype = "application/json"
	}
	apiURL := strings.TrimRight(c.URL, "/")
	if apiURL == "" {
		apiURL = "https://api.heroku.com"
	}
	req, err := http.NewRequest(method, apiURL+path, rbody)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/vnd.heroku+json; version=3")
	req.Header.Set("Request-Id", uuid.New())
	useragent := c.UserAgent
	if useragent == "" {
		useragent = DefaultUserAgent
	}
	req.Header.Set("User-Agent", useragent)
	if ctype != "" {
		req.Header.Set("Content-Type", ctype)
	}
	req.SetBasicAuth(c.Username, c.Password)
	for _, h := range strings.Split(os.Getenv("HKHEADER"), "\n") {
		if i := strings.Index(h, ":"); i >= 0 {
			req.Header.Set(
				strings.TrimSpace(h[:i]),
				strings.TrimSpace(h[i+1:]),
			)
		}
	}
	return req, nil
}

// Sends a Heroku API request and decodes the response into v. As
// described in NewRequest(), the type of body determines how to
// encode the request body. As described in DoReq(), the type of
// v determines how to handle the response body.
func (c *Client) APIReq(v interface{}, meth, path string, body interface{}) error {
	req, err := c.NewRequest(meth, path, body)
	if err != nil {
		return err
	}
	return c.DoReq(req, v)
}

// Submits an HTTP request, checks its response, and deserializes
// the response into v. The type of v determines how to handle
// the response body:
//
//   nil        body is discarded
//   io.Writer  body is copied directly into v
//   else       body is decoded into v as json
//
func (c *Client) DoReq(req *http.Request, v interface{}) error {
	debug := os.Getenv("HKDEBUG") != ""
	if debug {
		dump, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			log.Println(err)
		} else {
			os.Stderr.Write(dump)
			os.Stderr.Write([]byte{'\n', '\n'})
		}
	}

	httpClient := c.HTTP
	if httpClient == nil {
		httpClient = http.DefaultClient
	}

	res, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if debug {
		dump, err := httputil.DumpResponse(res, true)
		if err != nil {
			log.Println(err)
		} else {
			os.Stderr.Write(dump)
			os.Stderr.Write([]byte{'\n'})
		}
	}
	if err = checkResp(res); err != nil {
		return err
	}
	switch t := v.(type) {
	case nil:
	case io.Writer:
		_, err = io.Copy(t, res.Body)
	default:
		err = json.NewDecoder(res.Body).Decode(v)
	}
	return err
}

func checkResp(res *http.Response) error {
	if res.StatusCode == 401 {
		return errors.New("Unauthorized")
	}
	if res.StatusCode == 403 {
		return errors.New("Unauthorized")
	}
	if res.StatusCode/100 != 2 { // 200, 201, 202, etc
		return errors.New("Unexpected error: " + res.Status)
	}
	if msg := res.Header.Get("X-Heroku-Warning"); msg != "" {
		fmt.Fprintln(os.Stderr, strings.TrimSpace(msg))
	}
	return nil
}

type ListRange struct {
	Field      string
	Max        int
	Descending bool
	FirstId    string
	LastId     string
}

func (r *ListRange) SetHeader(req *http.Request) {
	var hdrval string
	if r.Field != "" {
		hdrval += r.Field + " "
	}
	hdrval += r.FirstId + ".." + r.LastId
	if r.Max != 0 {
		hdrval += fmt.Sprintf("; max=%d", r.Max)
		if r.Descending {
			hdrval += ", "
		}
	}

	if r.Descending {
		hdrval += ", order=desc"
	}

	req.Header.Set("Range", hdrval)
	return
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lzw implements the Lempel-Ziv-Welch compressed data format,
// described in T. A. Welch, ``A Technique for High-Performance Data
// Compression'', Computer, 17(6) (June 1984), pp 8-19.
//
// In particular, it implements LZW as used by the TIFF file format, including
// an "off by one" algorithmic difference when compared to standard LZW.
package lzw // import "golang.org/x/image/tiff/lzw"

/*
This file was branched from src/pkg/compress/lzw/reader.go in the
standard library. Differences from the original are marked with "NOTE".

The tif_lzw.c file in the libtiff C library has this comment:

----
The 5.0 spec describes a different algorithm than Aldus
implements.  Specifically, Aldus does code length transitions
one code earlier than should be done (for real LZW).
Earlier versions of this library implemented the correct
LZW algorithm, but emitted codes in a bit order opposite
to the TIFF spec.  Thus, to maintain compatibility w/ Aldus
we interpret MSB-LSB ordered codes to be images written w/
old versions of this library, but otherwise adhere to the
Aldus "off by one" algorithm.
----

The Go code doesn't read (invalid) TIFF files written by old versions of
libtiff, but the LZW algorithm in this package still differs from the one in
Go's standard package library to accomodate this "off by one" in valid TIFFs.
*/

import (
	"bufio"
	"errors"
	"fmt"
	"io"
)

// Order specifies the bit ordering in an LZW data stream.
type Order int

const (
	// LSB means Least Significant Bits first, as used in the GIF file format.
	LSB Order = iota
	// MSB means Most Significant Bits first, as used in the TIFF and PDF
	// file formats.
	MSB
)

const (
	maxWidth           = 12
	decoderInvalidCode = 0xffff
	flushBuffer        = 1 << maxWidth
)

// decoder is the state from which the readXxx method converts a byte
// stream into a code stream.
type decoder struct {
	r        io.ByteReader
	bits     uint32
	nBits    uint
	width    uint
	read     func(*decoder) (uint16, error) // readLSB or readMSB
	litWidth int                            // width in bits of literal codes
	err      error

	// The first 1<<litWidth codes are literal codes.
	// The next two codes mean clear and EOF.
	// Other valid codes are in the range [lo, hi] where lo := clear + 2,
	// with the upper bound incrementing on each code seen.
	// overflow is the code at which hi overflows the code width. NOTE: TIFF's LZW is "off by one".
	// last is the most recently seen code, or decoderInvalidCode.
	clear, eof, hi, overflow, last uint16

	// Each code c in [lo, hi] expands to two or more bytes. For c != hi:
	//   suffix[c] is the last of these bytes.
	//   prefix[c] is the code for all but the last byte.
	//   This code can either be a literal code or another code in [lo, c).
	// The c == hi case is a special case.
	suffix [1 << maxWidth]uint8
	prefix [1 << maxWidth]uint16

	// output is the temporary output buffer.
	// Literal codes are accumulated from the start of the buffer.
	// Non-literal codes decode to a sequence of suffixes that are first
	// written right-to-left from the end of the buffer before being copied
	// to the start of the buffer.
	// It is flushed when it contains >= 1<<maxWidth bytes,
	// so that there is always room to decode an entire code.
	output [2 * 1 << maxWidth]byte
	o      int    // write index into output
	toRead []byte // bytes to return from Read
}

// readLSB returns the next code for "Least Significant Bits first" data.
func (d *decoder) readLSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << d.nBits
		d.nBits += 8
	}
	code := uint16(d.bits & (1<<d.width - 1))
	d.bits >>= d.width
	d.nBits -= d.width
	return code, nil
}

// readMSB returns the next code for "Most Significant Bits first" data.
func (d *decoder) readMSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << (24 - d.nBits)
		d.nBits += 8
	}
	code := uint16(d.bits >> (32 - d.width))
	d.bits <<= d.width
	d.nBits -= d.width
	return code, nil
}

func (d *decoder) Read(b []byte) (int, error) {
	for {
		if len(d.toRead) > 0 {
			n := copy(b, d.toRead)
			d.toRead = d.toRead[n:]
			return n, nil
		}
		if d.err != nil {
			return 0, d.err
		}
		d.decode()
	}
}

// decode decompresses bytes from r and leaves them in d.toRead.
// read specifies how to decode bytes into codes.
// litWidth is the width in bits of literal codes.
func (d *decoder) decode() {
	// Loop over the code stream, converting codes into decompressed bytes.
	for {
		code, err := d.read(d)
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			d.err = err
			return
		}
		switch {
		case code < d.clear:
			// We have a literal code.
			d.output[d.o] = uint8(code)
			d.o++
			if d.last != decoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(code)
				d.prefix[d.hi] = d.last
			}
		case code == d.clear:
			d.width = 1 + uint(d.litWidth)
			d.hi = d.eof
			d.overflow = 1 << d.width
			d.last = decoderInvalidCode
			continue
		case code == d.eof:
			d.flush()
			d.err = io.EOF
			return
		case code <= d.hi:
			c, i := code, len(d.output)-1
			if code == d.hi {
				// code == hi is a special case which expands to the last expansion
				// followed by the head of the last expansion. To find the head, we walk
				// the prefix chain until we find a literal code.
				c = d.last
				for c >= d.clear {
					c = d.prefix[c]
				}
				d.output[i] = uint8(c)
				i--
				c = d.last
			}
			// Copy the suffix chain into output and then write that to w.
			for c >= d.clear {
				d.output[i] = d.suffix[c]
				i--
				c = d.prefix[c]
			}
			d.output[i] = uint8(c)
			d.o += copy(d.output[d.o:], d.output[i:])
			if d.last != decoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(c)
				d.prefix[d.hi] = d.last
			}
		default:
			d.err = errors.New("lzw: invalid code")
			return
		}
		d.last, d.hi = code, d.hi+1
		if d.hi+1 >= d.overflow { // NOTE: the "+1" is where TIFF's LZW differs from the standard algorithm.
			if d.width == maxWidth {
				d.last = decoderInvalidCode
			} else {
				d.width++
				d.overflow <<= 1
			}
		}
		if d.o >= flushBuffer {
			d.flush()
			return
		}
	}
}

func (d *decoder) flush() {
	d.toRead = d.output[:d.o]
	d.o = 0
}

var errClosed = errors.New("lzw: reader/writer is closed")

func (d *decoder) Close() error {
	d.err = errClosed // in case any Reads come along
	return nil
}

// NewReader creates a new io.ReadCloser.
// Reads from the returned io.ReadCloser read and decompress data from r.
// If r does not also implement io.ByteReader,
// the decompressor may read more data than necessary from r.
// It is the caller's responsibility to call Close on the ReadCloser when
// finished reading.
// The number of bits to use for literal codes, litWidth, must be in the
// range [2,8] and is typically 8. It must equal the litWidth
// used during compression.
func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser {
	d := new(decoder)
	switch order {
	case LSB:
		d.read = (*decoder).readLSB
	case MSB:
		d.read = (*decoder).readMSB
	default:
		d.err = errors.New("lzw: unknown order")
		return d
	}
	if litWidth < 2 || 8 < litWidth {
		d.err = fmt.Errorf("lzw: litWidth %d out of range", litWidth)
		return d
	}
	if br, ok := r.(io.ByteReader); ok {
		d.r = br
	} else {
		d.r = bufio.NewReader(r)
	}
	d.litWidth = litWidth
	d.width = 1 + uint(litWidth)
	d.clear = uint16(1) << uint(litWidth)
	d.eof, d.hi = d.clear+1, d.clear+1
	d.overflow = uint16(1) << d.width
	d.last = decoderInvalidCode

	return d
}
<commit_msg>tiff/lzw: sync (yet again) to the upstream lzw in the stdlib.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lzw implements the Lempel-Ziv-Welch compressed data format,
// described in T. A. Welch, ``A Technique for High-Performance Data
// Compression'', Computer, 17(6) (June 1984), pp 8-19.
//
// In particular, it implements LZW as used by the TIFF file format, including
// an "off by one" algorithmic difference when compared to standard LZW.
package lzw // import "golang.org/x/image/tiff/lzw"

/*
This file was branched from src/pkg/compress/lzw/reader.go in the
standard library. Differences from the original are marked with "NOTE".

The tif_lzw.c file in the libtiff C library has this comment:

----
The 5.0 spec describes a different algorithm than Aldus
implements.  Specifically, Aldus does code length transitions
one code earlier than should be done (for real LZW).
Earlier versions of this library implemented the correct
LZW algorithm, but emitted codes in a bit order opposite
to the TIFF spec.  Thus, to maintain compatibility w/ Aldus
we interpret MSB-LSB ordered codes to be images written w/
old versions of this library, but otherwise adhere to the
Aldus "off by one" algorithm.
----

The Go code doesn't read (invalid) TIFF files written by old versions of
libtiff, but the LZW algorithm in this package still differs from the one in
Go's standard package library to accomodate this "off by one" in valid TIFFs.
*/

import (
	"bufio"
	"errors"
	"fmt"
	"io"
)

// Order specifies the bit ordering in an LZW data stream.
type Order int

const (
	// LSB means Least Significant Bits first, as used in the GIF file format.
	LSB Order = iota
	// MSB means Most Significant Bits first, as used in the TIFF and PDF
	// file formats.
	MSB
)

const (
	maxWidth           = 12
	decoderInvalidCode = 0xffff
	flushBuffer        = 1 << maxWidth
)

// decoder is the state from which the readXxx method converts a byte
// stream into a code stream.
type decoder struct {
	r        io.ByteReader
	bits     uint32
	nBits    uint
	width    uint
	read     func(*decoder) (uint16, error) // readLSB or readMSB
	litWidth int                            // width in bits of literal codes
	err      error

	// The first 1<<litWidth codes are literal codes.
	// The next two codes mean clear and EOF.
	// Other valid codes are in the range [lo, hi] where lo := clear + 2,
	// with the upper bound incrementing on each code seen.
	// overflow is the code at which hi overflows the code width. NOTE: TIFF's LZW is "off by one".
	// last is the most recently seen code, or decoderInvalidCode.
	clear, eof, hi, overflow, last uint16

	// Each code c in [lo, hi] expands to two or more bytes. For c != hi:
	//   suffix[c] is the last of these bytes.
	//   prefix[c] is the code for all but the last byte.
	//   This code can either be a literal code or another code in [lo, c).
	// The c == hi case is a special case.
	suffix [1 << maxWidth]uint8
	prefix [1 << maxWidth]uint16

	// output is the temporary output buffer.
	// Literal codes are accumulated from the start of the buffer.
	// Non-literal codes decode to a sequence of suffixes that are first
	// written right-to-left from the end of the buffer before being copied
	// to the start of the buffer.
	// It is flushed when it contains >= 1<<maxWidth bytes,
	// so that there is always room to decode an entire code.
	output [2 * 1 << maxWidth]byte
	o      int    // write index into output
	toRead []byte // bytes to return from Read
}

// readLSB returns the next code for "Least Significant Bits first" data.
func (d *decoder) readLSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << d.nBits
		d.nBits += 8
	}
	code := uint16(d.bits & (1<<d.width - 1))
	d.bits >>= d.width
	d.nBits -= d.width
	return code, nil
}

// readMSB returns the next code for "Most Significant Bits first" data.
func (d *decoder) readMSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << (24 - d.nBits)
		d.nBits += 8
	}
	code := uint16(d.bits >> (32 - d.width))
	d.bits <<= d.width
	d.nBits -= d.width
	return code, nil
}

func (d *decoder) Read(b []byte) (int, error) {
	for {
		if len(d.toRead) > 0 {
			n := copy(b, d.toRead)
			d.toRead = d.toRead[n:]
			return n, nil
		}
		if d.err != nil {
			return 0, d.err
		}
		d.decode()
	}
}

// decode decompresses bytes from r and leaves them in d.toRead.
// read specifies how to decode bytes into codes.
// litWidth is the width in bits of literal codes.
func (d *decoder) decode() {
	// Loop over the code stream, converting codes into decompressed bytes.
	for {
		code, err := d.read(d)
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			d.err = err
			d.flush()
			return
		}
		switch {
		case code < d.clear:
			// We have a literal code.
			d.output[d.o] = uint8(code)
			d.o++
			if d.last != decoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(code)
				d.prefix[d.hi] = d.last
			}
		case code == d.clear:
			d.width = 1 + uint(d.litWidth)
			d.hi = d.eof
			d.overflow = 1 << d.width
			d.last = decoderInvalidCode
			continue
		case code == d.eof:
			d.flush()
			d.err = io.EOF
			return
		case code <= d.hi:
			c, i := code, len(d.output)-1
			if code == d.hi {
				// code == hi is a special case which expands to the last expansion
				// followed by the head of the last expansion. To find the head, we walk
				// the prefix chain until we find a literal code.
				c = d.last
				for c >= d.clear {
					c = d.prefix[c]
				}
				d.output[i] = uint8(c)
				i--
				c = d.last
			}
			// Copy the suffix chain into output and then write that to w.
			for c >= d.clear {
				d.output[i] = d.suffix[c]
				i--
				c = d.prefix[c]
			}
			d.output[i] = uint8(c)
			d.o += copy(d.output[d.o:], d.output[i:])
			if d.last != decoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(c)
				d.prefix[d.hi] = d.last
			}
		default:
			d.err = errors.New("lzw: invalid code")
			d.flush()
			return
		}
		d.last, d.hi = code, d.hi+1
		if d.hi+1 >= d.overflow { // NOTE: the "+1" is where TIFF's LZW differs from the standard algorithm.
			if d.width == maxWidth {
				d.last = decoderInvalidCode
			} else {
				d.width++
				d.overflow <<= 1
			}
		}
		if d.o >= flushBuffer {
			d.flush()
			return
		}
	}
}

func (d *decoder) flush() {
	d.toRead = d.output[:d.o]
	d.o = 0
}

var errClosed = errors.New("lzw: reader/writer is closed")

func (d *decoder) Close() error {
	d.err = errClosed // in case any Reads come along
	return nil
}

// NewReader creates a new io.ReadCloser.
// Reads from the returned io.ReadCloser read and decompress data from r.
// If r does not also implement io.ByteReader,
// the decompressor may read more data than necessary from r.
// It is the caller's responsibility to call Close on the ReadCloser when
// finished reading.
// The number of bits to use for literal codes, litWidth, must be in the
// range [2,8] and is typically 8. It must equal the litWidth
// used during compression.
func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser {
	d := new(decoder)
	switch order {
	case LSB:
		d.read = (*decoder).readLSB
	case MSB:
		d.read = (*decoder).readMSB
	default:
		d.err = errors.New("lzw: unknown order")
		return d
	}
	if litWidth < 2 || 8 < litWidth {
		d.err = fmt.Errorf("lzw: litWidth %d out of range", litWidth)
		return d
	}
	if br, ok := r.(io.ByteReader); ok {
		d.r = br
	} else {
		d.r = bufio.NewReader(r)
	}
	d.litWidth = litWidth
	d.width = 1 + uint(litWidth)
	d.clear = uint16(1) << uint(litWidth)
	d.eof, d.hi = d.clear+1, d.clear+1
	d.overflow = uint16(1) << d.width
	d.last = decoderInvalidCode

	return d
}
<|endoftext|>


<commit_before>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"github.com/pingcap/errors"
	"github.com/pingcap/parser/model"
	"github.com/pingcap/tidb/expression"
	"github.com/pingcap/tidb/expression/aggregation"
	"github.com/pingcap/tidb/sessionctx"
	"github.com/pingcap/tidb/table"
	"github.com/pingcap/tidb/tablecodec"
	"github.com/pingcap/tidb/util/codec"
	"github.com/pingcap/tidb/util/ranger"
	"github.com/pingcap/tipb/go-tipb"
)

// ToPB implements PhysicalPlan ToPB interface.
func (p *basePhysicalPlan) ToPB(_ sessionctx.Context) (*tipb.Executor, error) {
	return nil, errors.Errorf("plan %s fails converts to PB", p.basePlan.ExplainID())
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalHashAgg) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	sc := ctx.GetSessionVars().StmtCtx
	client := ctx.GetClient()
	aggExec := &tipb.Aggregation{
		GroupBy: expression.ExpressionsToPBList(sc, p.GroupByItems, client),
	}
	for _, aggFunc := range p.AggFuncs {
		aggExec.AggFunc = append(aggExec.AggFunc, aggregation.AggFuncToPBExpr(sc, client, aggFunc))
	}
	return &tipb.Executor{Tp: tipb.ExecType_TypeAggregation, Aggregation: aggExec}, nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalStreamAgg) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	sc := ctx.GetSessionVars().StmtCtx
	client := ctx.GetClient()
	aggExec := &tipb.Aggregation{
		GroupBy: expression.ExpressionsToPBList(sc, p.GroupByItems, client),
	}
	for _, aggFunc := range p.AggFuncs {
		aggExec.AggFunc = append(aggExec.AggFunc, aggregation.AggFuncToPBExpr(sc, client, aggFunc))
	}
	return &tipb.Executor{Tp: tipb.ExecType_TypeStreamAgg, Aggregation: aggExec}, nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalSelection) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	sc := ctx.GetSessionVars().StmtCtx
	client := ctx.GetClient()
	selExec := &tipb.Selection{
		Conditions: expression.ExpressionsToPBList(sc, p.Conditions, client),
	}
	return &tipb.Executor{Tp: tipb.ExecType_TypeSelection, Selection: selExec}, nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalTopN) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	sc := ctx.GetSessionVars().StmtCtx
	client := ctx.GetClient()
	topNExec := &tipb.TopN{
		Limit: p.Count,
	}
	for _, item := range p.ByItems {
		topNExec.OrderBy = append(topNExec.OrderBy, expression.SortByItemToPB(sc, client, item.Expr, item.Desc))
	}
	return &tipb.Executor{Tp: tipb.ExecType_TypeTopN, TopN: topNExec}, nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalLimit) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	limitExec := &tipb.Limit{
		Limit: p.Count,
	}
	return &tipb.Executor{Tp: tipb.ExecType_TypeLimit, Limit: limitExec}, nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalTableScan) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	tsExec := &tipb.TableScan{
		TableId: p.Table.ID,
		Columns: model.ColumnsToProto(p.Columns, p.Table.PKIsHandle),
		Desc:    p.Desc,
	}
	err := SetPBColumnsDefaultValue(ctx, tsExec.Columns, p.Columns)
	return &tipb.Executor{Tp: tipb.ExecType_TypeTableScan, TblScan: tsExec}, err
}

// checkCoverIndex checks whether we can pass unique info to TiKV. We should push it if and only if the length of
// range and index are equal.
func checkCoverIndex(idx *model.IndexInfo, ranges []*ranger.Range) bool {
	// If the index is (c1, c2) but the query range only contains c1, it is not a unique get.
	if !idx.Unique {
		return false
	}
	for _, rg := range ranges {
		if len(rg.LowVal) != len(idx.Columns) {
			return false
		}
	}
	return true
}

func findColumnInfoByID(infos []*model.ColumnInfo, id int64) *model.ColumnInfo {
	for _, info := range infos {
		if info.ID == id {
			return info
		}
	}
	return nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalIndexScan) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	columns := make([]*model.ColumnInfo, 0, p.schema.Len())
	tableColumns := p.Table.Cols()
	for _, col := range p.schema.Columns {
		if col.ID == model.ExtraHandleID {
			columns = append(columns, model.NewExtraHandleColInfo())
		} else {
			columns = append(columns, findColumnInfoByID(tableColumns, col.ID))
		}
	}
	idxExec := &tipb.IndexScan{
		TableId: p.Table.ID,
		IndexId: p.Index.ID,
		Columns: model.ColumnsToProto(columns, p.Table.PKIsHandle),
		Desc:    p.Desc,
	}
	unique := checkCoverIndex(p.Index, p.Ranges)
	idxExec.Unique = &unique
	return &tipb.Executor{Tp: tipb.ExecType_TypeIndexScan, IdxScan: idxExec}, nil
}

// SetPBColumnsDefaultValue sets the default values of tipb.ColumnInfos.
func SetPBColumnsDefaultValue(ctx sessionctx.Context, pbColumns []*tipb.ColumnInfo, columns []*model.ColumnInfo) error {
	for i, c := range columns {
		// For virtual columns, we set their default values to NULL so that TiKV will return NULL properly,
		// They real values will be compute later.
		if c.IsGenerated() && !c.GeneratedStored {
			pbColumns[i].DefaultVal = []byte{codec.NilFlag}
		}
		if c.OriginDefaultValue == nil {
			continue
		}

		sessVars := ctx.GetSessionVars()
		originStrict := sessVars.StrictSQLMode
		sessVars.StrictSQLMode = false
		d, err := table.GetColOriginDefaultValue(ctx, c)
		sessVars.StrictSQLMode = originStrict
		if err != nil {
			return err
		}

		pbColumns[i].DefaultVal, err = tablecodec.EncodeValue(sessVars.StmtCtx, nil, d)
		if err != nil {
			return err
		}
	}
	return nil
}

// SupportStreaming returns true if a pushed down operation supports using coprocessor streaming API.
// Note that this function handle pushed down physical plan only! It's called in constructDAGReq.
// Some plans are difficult (if possible) to implement streaming, and some are pointless to do so.
// TODO: Support more kinds of physical plan.
func SupportStreaming(p PhysicalPlan) bool {
	switch p.(type) {
	case *PhysicalIndexScan, *PhysicalSelection, *PhysicalTableScan:
		return true
	}
	return false
}
<commit_msg>planner: set the partition id to table id in DAG request (#14745)<commit_after>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"github.com/pingcap/errors"
	"github.com/pingcap/parser/model"
	"github.com/pingcap/tidb/expression"
	"github.com/pingcap/tidb/expression/aggregation"
	"github.com/pingcap/tidb/sessionctx"
	"github.com/pingcap/tidb/table"
	"github.com/pingcap/tidb/tablecodec"
	"github.com/pingcap/tidb/util/codec"
	"github.com/pingcap/tidb/util/ranger"
	"github.com/pingcap/tipb/go-tipb"
)

// ToPB implements PhysicalPlan ToPB interface.
func (p *basePhysicalPlan) ToPB(_ sessionctx.Context) (*tipb.Executor, error) {
	return nil, errors.Errorf("plan %s fails converts to PB", p.basePlan.ExplainID())
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalHashAgg) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	sc := ctx.GetSessionVars().StmtCtx
	client := ctx.GetClient()
	aggExec := &tipb.Aggregation{
		GroupBy: expression.ExpressionsToPBList(sc, p.GroupByItems, client),
	}
	for _, aggFunc := range p.AggFuncs {
		aggExec.AggFunc = append(aggExec.AggFunc, aggregation.AggFuncToPBExpr(sc, client, aggFunc))
	}
	return &tipb.Executor{Tp: tipb.ExecType_TypeAggregation, Aggregation: aggExec}, nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalStreamAgg) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	sc := ctx.GetSessionVars().StmtCtx
	client := ctx.GetClient()
	aggExec := &tipb.Aggregation{
		GroupBy: expression.ExpressionsToPBList(sc, p.GroupByItems, client),
	}
	for _, aggFunc := range p.AggFuncs {
		aggExec.AggFunc = append(aggExec.AggFunc, aggregation.AggFuncToPBExpr(sc, client, aggFunc))
	}
	return &tipb.Executor{Tp: tipb.ExecType_TypeStreamAgg, Aggregation: aggExec}, nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalSelection) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	sc := ctx.GetSessionVars().StmtCtx
	client := ctx.GetClient()
	selExec := &tipb.Selection{
		Conditions: expression.ExpressionsToPBList(sc, p.Conditions, client),
	}
	return &tipb.Executor{Tp: tipb.ExecType_TypeSelection, Selection: selExec}, nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalTopN) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	sc := ctx.GetSessionVars().StmtCtx
	client := ctx.GetClient()
	topNExec := &tipb.TopN{
		Limit: p.Count,
	}
	for _, item := range p.ByItems {
		topNExec.OrderBy = append(topNExec.OrderBy, expression.SortByItemToPB(sc, client, item.Expr, item.Desc))
	}
	return &tipb.Executor{Tp: tipb.ExecType_TypeTopN, TopN: topNExec}, nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalLimit) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	limitExec := &tipb.Limit{
		Limit: p.Count,
	}
	return &tipb.Executor{Tp: tipb.ExecType_TypeLimit, Limit: limitExec}, nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalTableScan) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	tsExec := &tipb.TableScan{
		TableId: p.Table.ID,
		Columns: model.ColumnsToProto(p.Columns, p.Table.PKIsHandle),
		Desc:    p.Desc,
	}
	if p.isPartition {
		tsExec.TableId = p.physicalTableID
	}
	err := SetPBColumnsDefaultValue(ctx, tsExec.Columns, p.Columns)
	return &tipb.Executor{Tp: tipb.ExecType_TypeTableScan, TblScan: tsExec}, err
}

// checkCoverIndex checks whether we can pass unique info to TiKV. We should push it if and only if the length of
// range and index are equal.
func checkCoverIndex(idx *model.IndexInfo, ranges []*ranger.Range) bool {
	// If the index is (c1, c2) but the query range only contains c1, it is not a unique get.
	if !idx.Unique {
		return false
	}
	for _, rg := range ranges {
		if len(rg.LowVal) != len(idx.Columns) {
			return false
		}
	}
	return true
}

func findColumnInfoByID(infos []*model.ColumnInfo, id int64) *model.ColumnInfo {
	for _, info := range infos {
		if info.ID == id {
			return info
		}
	}
	return nil
}

// ToPB implements PhysicalPlan ToPB interface.
func (p *PhysicalIndexScan) ToPB(ctx sessionctx.Context) (*tipb.Executor, error) {
	columns := make([]*model.ColumnInfo, 0, p.schema.Len())
	tableColumns := p.Table.Cols()
	for _, col := range p.schema.Columns {
		if col.ID == model.ExtraHandleID {
			columns = append(columns, model.NewExtraHandleColInfo())
		} else {
			columns = append(columns, findColumnInfoByID(tableColumns, col.ID))
		}
	}
	idxExec := &tipb.IndexScan{
		TableId: p.Table.ID,
		IndexId: p.Index.ID,
		Columns: model.ColumnsToProto(columns, p.Table.PKIsHandle),
		Desc:    p.Desc,
	}
	if p.isPartition {
		idxExec.TableId = p.physicalTableID
	}
	unique := checkCoverIndex(p.Index, p.Ranges)
	idxExec.Unique = &unique
	return &tipb.Executor{Tp: tipb.ExecType_TypeIndexScan, IdxScan: idxExec}, nil
}

// SetPBColumnsDefaultValue sets the default values of tipb.ColumnInfos.
func SetPBColumnsDefaultValue(ctx sessionctx.Context, pbColumns []*tipb.ColumnInfo, columns []*model.ColumnInfo) error {
	for i, c := range columns {
		// For virtual columns, we set their default values to NULL so that TiKV will return NULL properly,
		// They real values will be compute later.
		if c.IsGenerated() && !c.GeneratedStored {
			pbColumns[i].DefaultVal = []byte{codec.NilFlag}
		}
		if c.OriginDefaultValue == nil {
			continue
		}

		sessVars := ctx.GetSessionVars()
		originStrict := sessVars.StrictSQLMode
		sessVars.StrictSQLMode = false
		d, err := table.GetColOriginDefaultValue(ctx, c)
		sessVars.StrictSQLMode = originStrict
		if err != nil {
			return err
		}

		pbColumns[i].DefaultVal, err = tablecodec.EncodeValue(sessVars.StmtCtx, nil, d)
		if err != nil {
			return err
		}
	}
	return nil
}

// SupportStreaming returns true if a pushed down operation supports using coprocessor streaming API.
// Note that this function handle pushed down physical plan only! It's called in constructDAGReq.
// Some plans are difficult (if possible) to implement streaming, and some are pointless to do so.
// TODO: Support more kinds of physical plan.
func SupportStreaming(p PhysicalPlan) bool {
	switch p.(type) {
	case *PhysicalIndexScan, *PhysicalSelection, *PhysicalTableScan:
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers/dataGenerator"
	"time"

	"os"
	"runtime"

	"github.com/eaciit/orm"
	tk "github.com/eaciit/toolkit"
)

var (
	wd = func() string {
		d, _ := os.Getwd()
		return d + "/"
	}()
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	tk.Println("Starting the app..")

	start := time.Now().UTC()

	db, e := PrepareConnection()
	if e != nil {
		tk.Println(e)
	} else {
		base := new(BaseController)
		base.Ctx = orm.New(db)
		defer base.Ctx.Close()

		base.SetCollectionLatestTime()
		base.PrepareDataReff()

		// dependent Generate
		// new(UpdateScadaOemMinutes).GenerateDensity(base)    // step 0
		// new(UpdateOEMToScada).RunMapping(base)              // step 1
		// new(EventToAlarm).ConvertEventToAlarm(base)         // step 2
		// new(GenAlarmSummary).Generate(base)                 // step 3
		// new(GenDataPeriod).Generate(base)                   // step 4
		// new(GenScadaLast24).Generate(base)                  // step 5
		// new(GenScadaSummary).Generate(base)                 // step 6
		// new(GenScadaSummary).GenerateSummaryByFleet(base)   // step 7
		// new(GenScadaSummary).GenerateSummaryByProject(base) // step 8
		// new(GenScadaSummary).GenerateSummaryDaily(base)     // step 9
		// new(GenScadaSummary).GenWFAnalysisByProject(base)   // step 10
		// new(GenScadaSummary).GenWFAnalysisByTurbine1(base)  // step 11
		// new(GenScadaSummary).GenWFAnalysisByTurbine2(base)  // step 12

		// not dependent Generate
		new(DataAvailabilitySummary).ConvertDataAvailabilitySummary(base)
	}

	tk.Printf("DONE in %v Hrs \n", time.Now().UTC().Sub(start).Hours())
}
<commit_msg>add some comments<commit_after>package main

import (
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers/dataGenerator"
	"time"

	"os"
	"runtime"

	"github.com/eaciit/orm"
	tk "github.com/eaciit/toolkit"
)

var (
	wd = func() string {
		d, _ := os.Getwd()
		return d + "/"
	}()
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	tk.Println("Starting the app..")

	start := time.Now().UTC()

	db, e := PrepareConnection()
	if e != nil {
		tk.Println(e)
	} else {
		base := new(BaseController)
		base.Ctx = orm.New(db)
		defer base.Ctx.Close()

		base.SetCollectionLatestTime()
		base.PrepareDataReff()

		// dependent Generate
		// new(UpdateScadaOemMinutes).GenerateDensity(base)    // step 0
		// new(UpdateOEMToScada).RunMapping(base)              // step 1
		// new(EventToAlarm).ConvertEventToAlarm(base)         // step 2
		// new(GenAlarmSummary).Generate(base)                 // step 3
		// new(GenDataPeriod).Generate(base)                   // step 4
		// new(GenScadaLast24).Generate(base)                  // step 5
		// new(GenScadaSummary).Generate(base)                 // step 6
		// new(GenScadaSummary).GenerateSummaryByFleet(base)   // step 7
		// new(GenScadaSummary).GenerateSummaryByProject(base) // step 8
		// new(GenScadaSummary).GenerateSummaryDaily(base)     // step 9
		// new(GenScadaSummary).GenWFAnalysisByProject(base)   // step 10
		// new(GenScadaSummary).GenWFAnalysisByTurbine1(base)  // step 11
		// new(GenScadaSummary).GenWFAnalysisByTurbine2(base)  // step 12

		// not dependent Generate
		// new(DataAvailabilitySummary).ConvertDataAvailabilitySummary(base)

		/* data that need to copy:

		Alarm
		EventDown
		ScadaData
		ScadaDataOEM
		EventRaw
		GWFAnalysisBy***
		LatestDataPeriod -> just copy the data that changed
		rpt_***
		DataAvailability
		*/
	}

	tk.Printf("DONE in %v Hrs \n", time.Now().UTC().Sub(start).Hours())
}
<|endoftext|>


<commit_before>package timeout

import (
	"net"
	"net/http"
	"time"

	"github.com/getlantern/idletiming"
	"github.com/go-errors/errors"
)

const (
	DefaultConnectTimeout time.Duration = 30 * time.Second
	DefaultIdleTimeout                  = 60 * time.Second
)

func timeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (net.Conn, error) {
	return func(netw, addr string) (net.Conn, error) {
		conn, err := net.DialTimeout(netw, addr, cTimeout)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}
		idleConn := idletiming.Conn(conn, rwTimeout, func() {
			conn.Close()
		})
		return idleConn, nil
	}
}

func NewClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			Dial: timeoutDialer(connectTimeout, readWriteTimeout),
		},
	}
}

func NewDefaultClient() *http.Client {
	return NewClient(DefaultConnectTimeout, DefaultIdleTimeout)
}
<commit_msg>Use proxy from environment<commit_after>package timeout

import (
	"net"
	"net/http"
	"time"

	"github.com/getlantern/idletiming"
	"github.com/go-errors/errors"
)

const (
	DefaultConnectTimeout time.Duration = 30 * time.Second
	DefaultIdleTimeout                  = 60 * time.Second
)

func timeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (net.Conn, error) {
	return func(netw, addr string) (net.Conn, error) {
		conn, err := net.DialTimeout(netw, addr, cTimeout)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}
		idleConn := idletiming.Conn(conn, rwTimeout, func() {
			conn.Close()
		})
		return idleConn, nil
	}
}

func NewClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial:  timeoutDialer(connectTimeout, readWriteTimeout),
		},
	}
}

func NewDefaultClient() *http.Client {
	return NewClient(DefaultConnectTimeout, DefaultIdleTimeout)
}
<|endoftext|>


<commit_before>package honoka

import (
    "crypto/sha256"
    "encoding/hex"
    "encoding/json"
    "errors"
    "io/ioutil"
    "os"
    "path/filepath"
    "strconv"
    "time"

    homedir "github.com/mitchellh/go-homedir"
    "github.com/mitchellh/mapstructure"

    // for Debug
    // "fmt"
    // "github.com/davecgh/go-spew/spew"
)

type Client struct {
    Indexer IndexList
}

type IndexList map[string]Index

type Index struct {
    Key        string
    Bucket     string
    Expiration int64
}

type CleanResult struct {
    Bucket string
    Error  error
}

type UpdateFunc func() (interface{}, error)

var (
    Version = "0.0.1"
    BucketFileNotFound = errors.New("Not found specified bucket file")
    IndexFileNotFound  = errors.New("Not found specified index file")
    CacheIsExpired     = errors.New("specified cache is expired")
)

func New() (*Client, error) {
    idx, err := getIndexList()
    if err != nil {
        if err == IndexFileNotFound {
            idx = nil
        } else {
            return nil, err
        }
    }

    c := &Client{
        Indexer: idx,
    }
    return c, nil
}

func (c *Client) Get(key string, output interface{}) (interface{}, error) {
    if c.Expire(key) {
        return nil, CacheIsExpired
    }
    cache, err := c.GetJson(key)
    if err != nil {
        return nil, err
    }
    var result interface{}
    err = json.Unmarshal(cache, &result)
    if err != nil {
        return nil, err
    }
    err = mapstructure.WeakDecode(result, &output);
    return &output, err
}

func (c *Client) GetJson(key string) ([]byte, error) {
    if c.Expire(key) {
        return nil, CacheIsExpired
    }

    idx := c.Indexer[key]
    cache, err := getCacheFromBucket(idx.Bucket)
    if err != nil {
        return nil, err
    }
    return cache, nil
}

func (c *Client) Set(key string, val interface{}, expire int64) error {
    if ! c.Expire(key) {
        return nil
    }

    exp := createExpiration(expire)
    name := getBucketName(key, exp)
    _, err := createNewBucket(name, val)
    if err != nil {
        return err
    }
    var idx IndexList
    idx, err = getIndexList()
    if err != nil {
        if (err == IndexFileNotFound) {
            idx = map[string]Index{}
        } else {
            return err
        }
    }

    idx[key] = Index{
        Key:        key,
        Bucket:     name,
        Expiration: exp,
    }
    c.setIndexer(idx)

    return nil
}

func (c *Client) Update(key string, updater UpdateFunc, expire int64, output interface{}) (interface{}, error) {
    b, err := c.UpdateJson(key, updater, expire)
    if b != nil {
        var result interface{}
        e := json.Unmarshal(b, &result)
        if e != nil {
            return nil, e
        }

        e = mapstructure.WeakDecode(result, &output);
        if e != nil {
            return nil, e
        }
    }

    return output, err
}

func (c *Client) UpdateJson(key string, updater UpdateFunc, expire int64) ([]byte, error) {
    if ! c.Expire(key) {
        return c.GetJson(key)
    }

    val, err := updater()
    if err != nil {
        return nil, err
    }

    exp := createExpiration(expire)
    name := getBucketName(key, exp)
    jval, err := createNewBucket(name, val)
    if err != nil {
        return jval, err
    }
    var idx IndexList
    idx, err = getIndexList()
    if err != nil {
        idx = c.Indexer
    }

    idx[key] = Index{
        Key:        key,
        Bucket:     name,
        Expiration: exp,
    }
    c.setIndexer(idx)

    return jval, nil
}

func (c *Client) Delete(key string) error {
    idx := c.Indexer[key]
    path, err := getBucketPath(idx.Bucket)
    if err != nil {
        return err
    }
    if fileExists(path) {
        err = os.Remove(path)
        if err != nil {
            return err
        }
    }

    delete(c.Indexer, key)
    c.setIndexer(c.Indexer)
    return nil
}

func (c *Client) Expire(key string) bool {
    if nil == c.Indexer {
        return true
    }

    idx, exists := c.Indexer[key]
    if exists {
        if idx.Expiration <= time.Now().Unix() {
            c.Delete(key)
            return true
        } else {
            return false
        }
    }
    return true
}

func (c *Client) Outdated() ([]string, error) {
    idx, err := c.getIndexer(true)
    if err != nil {
        return nil, err
    }
    currents := make(map[string]string)
    for _, i := range idx {
        currents[i.Bucket] = ""
    }

    var list []string
    buckets, err := getBucketList()
    if err != nil {
        return nil, err
    }
    for _, bucket := range buckets {
        if _, exists := currents[bucket]; !exists {
            list = append(list, bucket)
        }
    }
    return list, nil
}

func (c *Client) Clean() ([]CleanResult, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return nil, err
    }
    list, err := c.Outdated()
    if err != nil {
        return nil, err
    }

    var result []CleanResult
    for _, bucket := range list {
        e := os.Remove(filepath.Join(bucketsDir, bucket))
        r := CleanResult{
            Bucket: bucket,
            Error:  e,
        }
        result = append(result, r)
    }
    return result, nil
}

func (c *Client) List() ([]Index, error) {
    idx, err := c.getIndexer(true)
    if err != nil {
        return nil, err
    }
    var list []Index
    for _, i := range idx {
        list = append(list, i)
    }

    return list, nil
}

func (c *Client) getIndexer(replace bool) (IndexList, error) {
    if replace || c.Indexer == nil {
        idx, err := getIndexList()
        if err != nil {
            return nil, err
        }
        c.Indexer = idx
    }
    return c.Indexer, nil
}

func (c *Client) setIndexer(indexes IndexList) error {
    idx, err := json.Marshal(indexes)
    if err != nil {
        return err
    }

    if err = updateIndexFile(idx); err != nil {
        return err
    }
    c.Indexer = indexes
    return nil
}

func getBucketsDirPath() (string, error) {
    home, err := homedir.Dir()
    if err != nil {
        return "", err
    }
    bucketsDir := filepath.Join(home, ".honoka", "buckets")
    os.MkdirAll(bucketsDir, 0700)
    return bucketsDir, err
}

func getBucketPath(bucketName string) (string, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return "", err
    }
    return filepath.Join(bucketsDir, bucketName), nil
}

func getCacheFromBucket(bucketName string) ([]byte, error) {
    path, err := getBucketPath(bucketName)
    if err != nil {
        return nil, err
    }
    if !fileExists(path) {
        return nil, BucketFileNotFound
    }
    return ioutil.ReadFile(path);
}

func getBucketList() ([]string, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return nil, err
    }
    files, err := ioutil.ReadDir(bucketsDir)
    var list []string
    for _, fi := range files {
        if !fi.IsDir() {
            filename := fi.Name()
            list = append(list, filename)
        }
    }
    return list, nil
}

func createNewBucket(name string, val interface{}) ([]byte, error) {
    jval, err := json.Marshal(val)
    if err != nil {
        return nil, err
    }
    path, err := getBucketPath(name)
    if err != nil {
        return jval, err
    }
    err = ioutil.WriteFile(path, jval, 0644)
    return jval, err
}

func getBucketName(key string, expiration int64) string {
    k := key + "." + strconv.FormatInt(expiration, 10)
    bytes := sha256.Sum256([]byte(k))
    return hex.EncodeToString(bytes[:])
}

func getIndexPath() (string, error) {
    home, err := homedir.Dir()
    if err != nil {
        return "", err
    }
    indexDir := filepath.Join(home, ".honoka")
    os.MkdirAll(indexDir, 0700)
    return filepath.Join(indexDir, "index"), err
}

func getIndexList() (IndexList, error) {
    b, err := getIndexFromFile()
    if err != nil {
        return nil, err
    }
    var list IndexList
    err = json.Unmarshal(b, &list)
    if  err != nil {
        return nil, err
    }
    return list, nil
}

func getIndexFromFile() ([]byte, error) {
    path, err := getIndexPath()
    if err != nil {
        return nil, err
    }
    if !fileExists(path) {
        return nil, IndexFileNotFound
    }
    return ioutil.ReadFile(path);
}

func updateIndexFile(indexes []byte) error {
    path, err := getIndexPath()
    if err != nil {
        return err
    }
    return ioutil.WriteFile(path, indexes, 0644);
}

func fileExists(filename string) bool {
    _, err := os.Stat(filename)
    return err == nil
}

func createExpiration(expire int64) int64 {
    return time.Now().Unix() + expire
}
<commit_msg>add docs<commit_after>package honoka

import (
    "crypto/sha256"
    "encoding/hex"
    "encoding/json"
    "errors"
    "io/ioutil"
    "os"
    "path/filepath"
    "strconv"
    "time"

    homedir "github.com/mitchellh/go-homedir"
    "github.com/mitchellh/mapstructure"

    // for Debug
    // "fmt"
    // "github.com/davecgh/go-spew/spew"
)

type Client struct {
    // Cache Index list
    Indexer IndexList
}

type IndexList map[string]Index

type Index struct {
    // The index key.
    Key        string

    // The bucket name that saved cache data.
    Bucket     string

    // The maximum elapsed time since the last file update.
    Expiration int64
}

type CleanResult struct {
    // The bucket name that saved cache data.
    Bucket string

    // Error when delete the specified bucket.
    Error  error
}

type UpdateFunc func() (interface{}, error)

var (
    Version = "0.0.1"
    BucketFileNotFound = errors.New("Not found specified bucket file")
    IndexFileNotFound  = errors.New("Not found specified index file")
    CacheIsExpired     = errors.New("specified cache is expired")
)

// New is a function for making a new cache
func New() (*Client, error) {
    idx, err := getIndexList()
    if err != nil {
        if err == IndexFileNotFound {
            idx = nil
        } else {
            return nil, err
        }
    }

    c := &Client{
        Indexer: idx,
    }
    return c, nil
}

// Get is used to retrieve a cache by specified key.
func (c *Client) Get(key string, output interface{}) (interface{}, error) {
    if c.Expire(key) {
        return nil, CacheIsExpired
    }
    cache, err := c.GetJson(key)
    if err != nil {
        return nil, err
    }
    var result interface{}
    err = json.Unmarshal(cache, &result)
    if err != nil {
        return nil, err
    }
    err = mapstructure.WeakDecode(result, &output);
    return &output, err
}

// Get is used to retrieve a cache by specified key.
// Return value is JSON string
func (c *Client) GetJson(key string) ([]byte, error) {
    if c.Expire(key) {
        return nil, CacheIsExpired
    }

    idx := c.Indexer[key]
    cache, err := getCacheFromBucket(idx.Bucket)
    if err != nil {
        return nil, err
    }
    return cache, nil
}

// Get is used to create a cache if specified key has not used yet.
func (c *Client) Set(key string, val interface{}, expire int64) error {
    if ! c.Expire(key) {
        return nil
    }

    exp := createExpiration(expire)
    name := getBucketName(key, exp)
    _, err := createNewBucket(name, val)
    if err != nil {
        return err
    }
    var idx IndexList
    idx, err = getIndexList()
    if err != nil {
        if (err == IndexFileNotFound) {
            idx = map[string]Index{}
        } else {
            return err
        }
    }

    idx[key] = Index{
        Key:        key,
        Bucket:     name,
        Expiration: exp,
    }
    c.setIndexer(idx)

    return nil
}

// Update calls the cache update function on the cached data.
func (c *Client) Update(key string, updater UpdateFunc, expire int64, output interface{}) (interface{}, error) {
    b, err := c.UpdateJson(key, updater, expire)
    if b != nil {
        var result interface{}
        e := json.Unmarshal(b, &result)
        if e != nil {
            return nil, e
        }

        e = mapstructure.WeakDecode(result, &output);
        if e != nil {
            return nil, e
        }
    }

    return output, err
}

// Update calls the cache update function on the cached data.
// Return value is JSON string.
func (c *Client) UpdateJson(key string, updater UpdateFunc, expire int64) ([]byte, error) {
    if ! c.Expire(key) {
        return c.GetJson(key)
    }

    val, err := updater()
    if err != nil {
        return nil, err
    }

    exp := createExpiration(expire)
    name := getBucketName(key, exp)
    jval, err := createNewBucket(name, val)
    if err != nil {
        return jval, err
    }
    var idx IndexList
    idx, err = getIndexList()
    if err != nil {
        idx = c.Indexer
    }

    idx[key] = Index{
        Key:        key,
        Bucket:     name,
        Expiration: exp,
    }
    c.setIndexer(idx)

    return jval, nil
}

// Delete is used to delete a cache by specified key.
func (c *Client) Delete(key string) error {
    idx := c.Indexer[key]
    path, err := getBucketPath(idx.Bucket)
    if err != nil {
        return err
    }
    if fileExists(path) {
        err = os.Remove(path)
        if err != nil {
            return err
        }
    }

    delete(c.Indexer, key)
    c.setIndexer(c.Indexer)
    return nil
}

// Expire is a predicate which determines if the cache should be updated.
func (c *Client) Expire(key string) bool {
    if nil == c.Indexer {
        return true
    }

    idx, exists := c.Indexer[key]
    if exists {
        if idx.Expiration <= time.Now().Unix() {
            c.Delete(key)
            return true
        } else {
            return false
        }
    }
    return true
}

// Outdated is used to retrive no-indexed bucket.
func (c *Client) Outdated() ([]string, error) {
    idx, err := c.getIndexer(true)
    if err != nil {
        return nil, err
    }
    currents := make(map[string]string)
    for _, i := range idx {
        currents[i.Bucket] = ""
    }

    var list []string
    buckets, err := getBucketList()
    if err != nil {
        return nil, err
    }
    for _, bucket := range buckets {
        if _, exists := currents[bucket]; !exists {
            list = append(list, bucket)
        }
    }
    return list, nil
}

// Clean is used to delete no-indexed bucket.
func (c *Client) Clean() ([]CleanResult, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return nil, err
    }
    list, err := c.Outdated()
    if err != nil {
        return nil, err
    }

    var result []CleanResult
    for _, bucket := range list {
        e := os.Remove(filepath.Join(bucketsDir, bucket))
        r := CleanResult{
            Bucket: bucket,
            Error:  e,
        }
        result = append(result, r)
    }
    return result, nil
}

// List is used to retrive cache indexes.
func (c *Client) List() ([]Index, error) {
    idx, err := c.getIndexer(true)
    if err != nil {
        return nil, err
    }
    var list []Index
    for _, i := range idx {
        list = append(list, i)
    }

    return list, nil
}

func (c *Client) getIndexer(replace bool) (IndexList, error) {
    if replace || c.Indexer == nil {
        idx, err := getIndexList()
        if err != nil {
            return nil, err
        }
        c.Indexer = idx
    }
    return c.Indexer, nil
}

func (c *Client) setIndexer(indexes IndexList) error {
    idx, err := json.Marshal(indexes)
    if err != nil {
        return err
    }

    if err = updateIndexFile(idx); err != nil {
        return err
    }
    c.Indexer = indexes
    return nil
}

func getBucketsDirPath() (string, error) {
    home, err := homedir.Dir()
    if err != nil {
        return "", err
    }
    bucketsDir := filepath.Join(home, ".honoka", "buckets")
    os.MkdirAll(bucketsDir, 0700)
    return bucketsDir, err
}

func getBucketPath(bucketName string) (string, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return "", err
    }
    return filepath.Join(bucketsDir, bucketName), nil
}

func getCacheFromBucket(bucketName string) ([]byte, error) {
    path, err := getBucketPath(bucketName)
    if err != nil {
        return nil, err
    }
    if !fileExists(path) {
        return nil, BucketFileNotFound
    }
    return ioutil.ReadFile(path);
}

func getBucketList() ([]string, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return nil, err
    }
    files, err := ioutil.ReadDir(bucketsDir)
    var list []string
    for _, fi := range files {
        if !fi.IsDir() {
            filename := fi.Name()
            list = append(list, filename)
        }
    }
    return list, nil
}

func createNewBucket(name string, val interface{}) ([]byte, error) {
    jval, err := json.Marshal(val)
    if err != nil {
        return nil, err
    }
    path, err := getBucketPath(name)
    if err != nil {
        return jval, err
    }
    err = ioutil.WriteFile(path, jval, 0644)
    return jval, err
}

func getBucketName(key string, expiration int64) string {
    k := key + "." + strconv.FormatInt(expiration, 10)
    bytes := sha256.Sum256([]byte(k))
    return hex.EncodeToString(bytes[:])
}

func getIndexPath() (string, error) {
    home, err := homedir.Dir()
    if err != nil {
        return "", err
    }
    indexDir := filepath.Join(home, ".honoka")
    os.MkdirAll(indexDir, 0700)
    return filepath.Join(indexDir, "index"), err
}

func getIndexList() (IndexList, error) {
    b, err := getIndexFromFile()
    if err != nil {
        return nil, err
    }
    var list IndexList
    err = json.Unmarshal(b, &list)
    if  err != nil {
        return nil, err
    }
    return list, nil
}

func getIndexFromFile() ([]byte, error) {
    path, err := getIndexPath()
    if err != nil {
        return nil, err
    }
    if !fileExists(path) {
        return nil, IndexFileNotFound
    }
    return ioutil.ReadFile(path);
}

func updateIndexFile(indexes []byte) error {
    path, err := getIndexPath()
    if err != nil {
        return err
    }
    return ioutil.WriteFile(path, indexes, 0644);
}

func fileExists(filename string) bool {
    _, err := os.Stat(filename)
    return err == nil
}

func createExpiration(expire int64) int64 {
    return time.Now().Unix() + expire
}
<|endoftext|>


<commit_before>package tokenizer

import (
	"fmt"
	"strings"

	"github.com/ikawaha/kagome-dict/dict"
	"github.com/ikawaha/kagome/v2/tokenizer/lattice"
)

// TokenClass represents the token class.
type TokenClass lattice.NodeClass

const (
	// DUMMY represents the dummy token.
	DUMMY = TokenClass(lattice.DUMMY)
	// KNOWN represents the token in the dictionary.
	KNOWN = TokenClass(lattice.KNOWN)
	// UNKNOWN represents the token which is not in the dictionary.
	UNKNOWN = TokenClass(lattice.UNKNOWN)
	// USER represents the token in the user dictionary.
	USER = TokenClass(lattice.USER)
)

// String returns string representation of a token class.
func (c TokenClass) String() string {
	ret := ""
	switch c {
	case DUMMY:
		ret = "DUMMY"
	case KNOWN:
		ret = "KNOWN"
	case UNKNOWN:
		ret = "UNKNOWN"
	case USER:
		ret = "USER"
	}
	return ret
}

// Token represents a morph of a sentence.
type Token struct {
	Index    int
	ID       int
	Class    TokenClass
	Position int // byte position
	Start    int
	End      int
	Surface  string
	dict     *dict.Dict
	udict    *dict.UserDict
}

// Features returns contents of a token.
func (t Token) Features() []string {
	switch t.Class {
	case KNOWN:
		var c int
		if t.dict.Contents != nil {
			c = len(t.dict.Contents[t.ID])
		}
		features := make([]string, 0, len(t.dict.POSTable.POSs[t.ID])+c)
		for _, id := range t.dict.POSTable.POSs[t.ID] {
			features = append(features, t.dict.POSTable.NameList[id])
		}
		if t.dict.Contents != nil {
			features = append(features, t.dict.Contents[t.ID]...)
		}
		return features
	case UNKNOWN:
		features := make([]string, len(t.dict.UnkDict.Contents[t.ID]))
		for i := range t.dict.UnkDict.Contents[t.ID] {
			features[i] = t.dict.UnkDict.Contents[t.ID][i]
		}
		return features
	case USER:
		pos := t.udict.Contents[t.ID].Pos
		tokens := strings.Join(t.udict.Contents[t.ID].Tokens, "/")
		yomi := strings.Join(t.udict.Contents[t.ID].Yomi, "/")
		return []string{pos, tokens, yomi}
	}
	return nil
}

// FeatureAt returns the i th feature if exists.
func (t Token) FeatureAt(i int) (string, bool) {
	if i < 0 {
		return "", false
	}
	switch t.Class {
	case KNOWN:
		pos := t.dict.POSTable.POSs[t.ID]
		if i < len(pos) {
			id := pos[i]
			if int(id) > len(t.dict.POSTable.NameList) {
				return "", false
			}
			return t.dict.POSTable.NameList[id], true
		}
		i -= len(pos)
		if len(t.dict.Contents) <= t.ID {
			return "", false
		}
		c := t.dict.Contents[t.ID]
		if i >= len(c) {
			return "", false
		}
		return c[i], true
	case UNKNOWN:
		if len(t.dict.UnkDict.Contents) <= t.ID {
			return "", false
		}
		c := t.dict.UnkDict.Contents[t.ID]
		if i >= len(c) {
			return "", false
		}
		return c[i], true
	case USER:
		if len(t.udict.Contents) <= t.ID {
			return "", false
		}
		switch i {
		case 0:
			return t.udict.Contents[t.ID].Pos, true
		case 1:
			return strings.Join(t.udict.Contents[t.ID].Tokens, "/"), true
		case 2:
			return strings.Join(t.udict.Contents[t.ID].Yomi, "/"), true
		}
	}
	return "", false
}

// POS returns POS elements of features.
func (t Token) POS() []string {
	switch t.Class {
	case KNOWN:
		ret := make([]string, 0, len(t.dict.POSTable.POSs[t.ID]))
		for _, id := range t.dict.POSTable.POSs[t.ID] {
			ret = append(ret, t.dict.POSTable.NameList[id])
		}
		return ret
	case UNKNOWN:
		start := 0
		if v, ok := t.dict.UnkDict.ContentsMeta[dict.POSStartIndex]; ok {
			start = int(v)
		}
		end := 1
		if v, ok := t.dict.UnkDict.ContentsMeta[dict.POSHierarchy]; ok {
			end = start + int(v)
		}
		feature := t.dict.UnkDict.Contents[t.ID]
		if start >= end || end > len(feature) {
			return nil
		}
		ret := make([]string, 0, end-start)
		for i := start; i < end; i++ {
			ret = append(ret, feature[i])
		}
		return ret
	case USER:
		pos := t.udict.Contents[t.ID].Pos
		return []string{pos}
	}
	return nil
}

// InflectionalType returns the inflectional type feature if exists.
func (t Token) InflectionalType() (string, bool) {
	return t.pickupFromFeatures(dict.InflectionalType)
}

// InflectionalForm returns the inflectional form feature if exists.
func (t Token) InflectionalForm() (string, bool) {
	return t.pickupFromFeatures(dict.InflectionalForm)
}

// BaseForm returns the base form features if exists.
func (t Token) BaseForm() (string, bool) {
	return t.pickupFromFeatures(dict.BaseFormIndex)
}

// Reading returns the reading feature if exists.
func (t Token) Reading() (string, bool) {
	return t.pickupFromFeatures(dict.ReadingIndex)
}

// Pronunciation returns the pronunciation feature if exists.
func (t Token) Pronunciation() (string, bool) {
	return t.pickupFromFeatures(dict.PronunciationIndex)
}

func (t Token) pickupFromFeatures(key string) (string, bool) {
	var meta dict.ContentsMeta
	switch t.Class {
	case KNOWN:
		meta = t.dict.ContentsMeta
	case UNKNOWN:
		meta = t.dict.UnkDict.ContentsMeta
	}
	i, ok := meta[key]
	if !ok {
		return "", false
	}
	return t.FeatureAt(int(i))
}

// String returns a string representation of a token.
func (t Token) String() string {
	return fmt.Sprintf("%d:%q (%d: %d, %d) %v [%d]", t.Index, t.Surface, t.Position, t.Start, t.End, t.Class, t.ID)
}
<commit_msg>Fix equality of tokens<commit_after>package tokenizer

import (
	"fmt"
	"strings"

	"github.com/ikawaha/kagome-dict/dict"
	"github.com/ikawaha/kagome/v2/tokenizer/lattice"
)

// TokenClass represents the token class.
type TokenClass lattice.NodeClass

const (
	// DUMMY represents the dummy token.
	DUMMY = TokenClass(lattice.DUMMY)
	// KNOWN represents the token in the dictionary.
	KNOWN = TokenClass(lattice.KNOWN)
	// UNKNOWN represents the token which is not in the dictionary.
	UNKNOWN = TokenClass(lattice.UNKNOWN)
	// USER represents the token in the user dictionary.
	USER = TokenClass(lattice.USER)
)

// String returns string representation of a token class.
func (c TokenClass) String() string {
	ret := ""
	switch c {
	case DUMMY:
		ret = "DUMMY"
	case KNOWN:
		ret = "KNOWN"
	case UNKNOWN:
		ret = "UNKNOWN"
	case USER:
		ret = "USER"
	}
	return ret
}

// Token represents a morph of a sentence.
type Token struct {
	Index    int
	ID       int
	Class    TokenClass
	Position int // byte position
	Start    int
	End      int
	Surface  string
	dict     *dict.Dict
	udict    *dict.UserDict
}

// Features returns contents of a token.
func (t Token) Features() []string {
	switch t.Class {
	case KNOWN:
		var c int
		if t.dict.Contents != nil {
			c = len(t.dict.Contents[t.ID])
		}
		features := make([]string, 0, len(t.dict.POSTable.POSs[t.ID])+c)
		for _, id := range t.dict.POSTable.POSs[t.ID] {
			features = append(features, t.dict.POSTable.NameList[id])
		}
		if t.dict.Contents != nil {
			features = append(features, t.dict.Contents[t.ID]...)
		}
		return features
	case UNKNOWN:
		features := make([]string, len(t.dict.UnkDict.Contents[t.ID]))
		for i := range t.dict.UnkDict.Contents[t.ID] {
			features[i] = t.dict.UnkDict.Contents[t.ID][i]
		}
		return features
	case USER:
		pos := t.udict.Contents[t.ID].Pos
		tokens := strings.Join(t.udict.Contents[t.ID].Tokens, "/")
		yomi := strings.Join(t.udict.Contents[t.ID].Yomi, "/")
		return []string{pos, tokens, yomi}
	}
	return nil
}

// FeatureAt returns the i th feature if exists.
func (t Token) FeatureAt(i int) (string, bool) {
	if i < 0 {
		return "", false
	}
	switch t.Class {
	case KNOWN:
		pos := t.dict.POSTable.POSs[t.ID]
		if i < len(pos) {
			id := pos[i]
			if int(id) > len(t.dict.POSTable.NameList) {
				return "", false
			}
			return t.dict.POSTable.NameList[id], true
		}
		i -= len(pos)
		if len(t.dict.Contents) <= t.ID {
			return "", false
		}
		c := t.dict.Contents[t.ID]
		if i >= len(c) {
			return "", false
		}
		return c[i], true
	case UNKNOWN:
		if len(t.dict.UnkDict.Contents) <= t.ID {
			return "", false
		}
		c := t.dict.UnkDict.Contents[t.ID]
		if i >= len(c) {
			return "", false
		}
		return c[i], true
	case USER:
		if len(t.udict.Contents) <= t.ID {
			return "", false
		}
		switch i {
		case 0:
			return t.udict.Contents[t.ID].Pos, true
		case 1:
			return strings.Join(t.udict.Contents[t.ID].Tokens, "/"), true
		case 2:
			return strings.Join(t.udict.Contents[t.ID].Yomi, "/"), true
		}
	}
	return "", false
}

// POS returns POS elements of features.
func (t Token) POS() []string {
	switch t.Class {
	case KNOWN:
		ret := make([]string, 0, len(t.dict.POSTable.POSs[t.ID]))
		for _, id := range t.dict.POSTable.POSs[t.ID] {
			ret = append(ret, t.dict.POSTable.NameList[id])
		}
		return ret
	case UNKNOWN:
		start := 0
		if v, ok := t.dict.UnkDict.ContentsMeta[dict.POSStartIndex]; ok {
			start = int(v)
		}
		end := 1
		if v, ok := t.dict.UnkDict.ContentsMeta[dict.POSHierarchy]; ok {
			end = start + int(v)
		}
		feature := t.dict.UnkDict.Contents[t.ID]
		if start >= end || end > len(feature) {
			return nil
		}
		ret := make([]string, 0, end-start)
		for i := start; i < end; i++ {
			ret = append(ret, feature[i])
		}
		return ret
	case USER:
		pos := t.udict.Contents[t.ID].Pos
		return []string{pos}
	}
	return nil
}

// InflectionalType returns the inflectional type feature if exists.
func (t Token) InflectionalType() (string, bool) {
	return t.pickupFromFeatures(dict.InflectionalType)
}

// InflectionalForm returns the inflectional form feature if exists.
func (t Token) InflectionalForm() (string, bool) {
	return t.pickupFromFeatures(dict.InflectionalForm)
}

// BaseForm returns the base form features if exists.
func (t Token) BaseForm() (string, bool) {
	return t.pickupFromFeatures(dict.BaseFormIndex)
}

// Reading returns the reading feature if exists.
func (t Token) Reading() (string, bool) {
	return t.pickupFromFeatures(dict.ReadingIndex)
}

// Pronunciation returns the pronunciation feature if exists.
func (t Token) Pronunciation() (string, bool) {
	return t.pickupFromFeatures(dict.PronunciationIndex)
}

func (t Token) pickupFromFeatures(key string) (string, bool) {
	var meta dict.ContentsMeta
	switch t.Class {
	case KNOWN:
		meta = t.dict.ContentsMeta
	case UNKNOWN:
		meta = t.dict.UnkDict.ContentsMeta
	}
	i, ok := meta[key]
	if !ok {
		return "", false
	}
	return t.FeatureAt(int(i))
}

// String returns a string representation of a token.
func (t Token) String() string {
	return fmt.Sprintf("%d:%q (%d: %d, %d) %v [%d]", t.Index, t.Surface, t.Position, t.Start, t.End, t.Class, t.ID)
}

// Equal returns true if tokens are equal. This function compares values other than the `Index` field.
func (t Token) Equal(v Token) bool {
	return t.ID == v.ID &&
		t.Class == v.Class &&
		t.Surface == v.Surface
}
<|endoftext|>