<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package schema

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"camlistore.org/pkg/blob"
	"camlistore.org/pkg/singleflight"
	"camlistore.org/pkg/types"
)

const closedIndex = -1

var errClosed = errors.New("filereader is closed")

// A FileReader reads the bytes of "file" and "bytes" schema blobrefs.
type FileReader struct {
	// Immutable stuff:
	*io.SectionReader             // provides Read, etc.
	parent            *FileReader // or nil for sub-region readers to find the ssm map in getSuperset
	rootOff           int64       // this FileReader's offset from the root
	fetcher           blob.Fetcher
	ss                *superset
	size              int64 // total number of bytes

	sfg singleflight.Group // for loading blobrefs for ssm

	ssmmu sync.Mutex           // guards ssm
	ssm   map[string]*superset // blobref -> superset
}

// NewFileReader returns a new FileReader reading the contents of fileBlobRef,
// fetching blobs from fetcher.  The fileBlobRef must be of a "bytes" or "file"
// schema blob.
//
// The caller should call Close on the FileReader when done reading.
func NewFileReader(fetcher blob.Fetcher, fileBlobRef blob.Ref) (*FileReader, error) {
	// TODO(bradfitz): make this take a blobref.FetcherAt instead?
	// TODO(bradfitz): rename this into bytes reader? but for now it's still
	//                 named FileReader, but can also read a "bytes" schema.
	if !fileBlobRef.Valid() {
		return nil, errors.New("schema/filereader: NewFileReader blobref invalid")
	}
	rc, _, err := fetcher.Fetch(fileBlobRef)
	if err != nil {
		return nil, fmt.Errorf("schema/filereader: fetching file schema blob: %v", err)
	}
	defer rc.Close()
	ss, err := parseSuperset(rc)
	if err != nil {
		return nil, fmt.Errorf("schema/filereader: decoding file schema blob: %v", err)
	}
	if ss.Type != "file" && ss.Type != "bytes" {
		return nil, fmt.Errorf("schema/filereader: expected \"file\" or \"bytes\" schema blob, got %q", ss.Type)
	}
	fr, err := ss.NewFileReader(fetcher)
	if err != nil {
		return nil, fmt.Errorf("schema/filereader: creating FileReader for %s: %v", fileBlobRef, err)
	}
	return fr, nil
}

func (b *Blob) NewFileReader(fetcher blob.Fetcher) (*FileReader, error) {
	return b.ss.NewFileReader(fetcher)
}

// NewFileReader returns a new FileReader, reading bytes and blobs
// from the provided fetcher.
//
// NewFileReader does no fetch operation on the fetcher itself.  The
// fetcher is only used in subsequent read operations.
//
// An error is only returned if the type of the superset is not either
// "file" or "bytes".
func (ss *superset) NewFileReader(fetcher blob.Fetcher) (*FileReader, error) {
	if ss.Type != "file" && ss.Type != "bytes" {
		return nil, fmt.Errorf("schema/filereader: Superset not of type \"file\" or \"bytes\"")
	}
	size := int64(ss.SumPartsSize())
	fr := &FileReader{
		fetcher: fetcher,
		ss:      ss,
		size:    size,
		ssm:     make(map[string]*superset),
	}
	fr.SectionReader = io.NewSectionReader(fr, 0, size)
	return fr, nil
}

// LoadAllChunks causes all chunks of the file to be loaded as quickly
// as possible.  The contents are immediately discarded, so it is
// assumed that the fetcher is a caching fetcher.
func (fr *FileReader) LoadAllChunks() {
	offsetc := make(chan int64, 16)
	go func() {
		for off := range offsetc {
			go func(off int64) {
				rc, err := fr.readerForOffset(off)
				if err == nil {
					defer rc.Close()
					var b [1]byte
					rc.Read(b[:]) // fault in the blob
				}
			}(off)
		}
	}()
	go fr.GetChunkOffsets(offsetc)
}

// UnixMtime returns the file schema's UnixMtime field, or the zero value.
func (fr *FileReader) UnixMtime() time.Time {
	t, err := time.Parse(time.RFC3339, fr.ss.UnixMtime)
	if err != nil {
		return time.Time{}
	}
	return t
}

// FileName returns the file schema's filename, if any.
func (fr *FileReader) FileName() string { return fr.ss.FileName }

func (fr *FileReader) Close() error {
	// TODO: close cached blobs?
	return nil
}

var _ interface {
	io.ReaderAt
	io.Reader
	io.Closer
	Size() int64
} = (*FileReader)(nil)

func (fr *FileReader) ReadAt(p []byte, offset int64) (n int, err error) {
	if offset < 0 {
		return 0, errors.New("schema/filereader: negative offset")
	}
	if offset >= fr.Size() {
		return 0, io.EOF
	}
	want := len(p)
	for len(p) > 0 && err == nil {
		var rc io.ReadCloser
		rc, err = fr.readerForOffset(offset)
		if err != nil {
			return
		}
		var n1 int64 // never bigger than an int
		n1, err = io.CopyN(&sliceWriter{p}, rc, int64(len(p)))
		rc.Close()
		if err == io.EOF {
			err = nil
		}
		if n1 == 0 {
			break
		}
		p = p[n1:]
		offset += int64(n1)
		n += int(n1)
	}
	if n < want && err == nil {
		err = io.ErrUnexpectedEOF
	}
	return n, err
}

// GetChunkOffsets sends c each of the file's chunk offsets.
// The offsets are not necessarily sent in order, and all ranges of the file
// are not necessarily represented if the file contains zero holes.
// The channel c is closed before the function returns, regardless of error.
func (fr *FileReader) GetChunkOffsets(c chan<- int64) error {
	defer close(c)
	firstErrc := make(chan error, 1)
	return fr.sendPartsChunks(c, firstErrc, 0, fr.ss.Parts)
}

// firstErrc is a communication mechanism amongst all outstanding
// superset-fetching goroutines to see if anybody else has failed.  If
// so (a non-blocking read returns something), then the recursive call
// to sendPartsChunks is skipped, hopefully preventing unnecessary
// work.  Whenever a caller receives on firstErrc, it should also send
// back to it.  It's buffered.
func (fr *FileReader) sendPartsChunks(c chan<- int64, firstErrc chan error, off int64, parts []*BytesPart) error {
	var errcs []chan error
	for _, p := range parts {
		switch {
		case p.BlobRef.Valid() && p.BytesRef.Valid():
			return fmt.Errorf("part illegally contained both a blobRef and bytesRef")
		case !p.BlobRef.Valid() && !p.BytesRef.Valid():
			// Don't send
		case p.BlobRef.Valid():
			c <- off
		case p.BytesRef.Valid():
			errc := make(chan error, 1)
			errcs = append(errcs, errc)
			br := p.BytesRef
			go func(off int64) (err error) {
				defer func() {
					errc <- err
					if err != nil {
						select {
						case firstErrc <- err: // pump
						default:
						}
					}
				}()
				select {
				case err = <-firstErrc:
					// There was already an error elsewhere in the file.
					// Avoid doing more work.
					return
				default:
					ss, err := fr.getSuperset(br)
					if err != nil {
						return err
					}
					return fr.sendPartsChunks(c, firstErrc, off, ss.Parts)
				}
			}(off)
		}
		off += int64(p.Size)
	}

	var retErr error
	for _, errc := range errcs {
		if err := <-errc; err != nil && retErr == nil {
			retErr = err
		}
	}
	return retErr
}

type sliceWriter struct {
	dst []byte
}

func (sw *sliceWriter) Write(p []byte) (n int, err error) {
	n = copy(sw.dst, p)
	sw.dst = sw.dst[n:]
	return n, nil
}

var eofReader io.ReadCloser = ioutil.NopCloser(strings.NewReader(""))

func (fr *FileReader) rootReader() *FileReader {
	if fr.parent != nil {
		return fr.parent.rootReader()
	}
	return fr
}

func (fr *FileReader) getSuperset(br blob.Ref) (*superset, error) {
	if root := fr.rootReader(); root != fr {
		return root.getSuperset(br)
	}
	brStr := br.String()
	ssi, err := fr.sfg.Do(brStr, func() (interface{}, error) {
		fr.ssmmu.Lock()
		ss, ok := fr.ssm[brStr]
		fr.ssmmu.Unlock()
		if ok {
			return ss, nil
		}
		rc, _, err := fr.fetcher.Fetch(br)
		if err != nil {
			return nil, fmt.Errorf("schema/filereader: fetching file schema blob: %v", err)
		}
		defer rc.Close()
		ss, err = parseSuperset(rc)
		if err != nil {
			return nil, err
		}
		fr.ssmmu.Lock()
		defer fr.ssmmu.Unlock()
		fr.ssm[brStr] = ss
		return ss, nil
	})
	if err != nil {
		return nil, err
	}
	return ssi.(*superset), nil
}

var debug = os.Getenv("CAMLI_DEBUG") != ""

// readerForOffset returns a ReadCloser that reads some number of bytes and then EOF
// from the provided offset.  Seeing EOF doesn't mean the end of the whole file; just the
// chunk at that offset.  The caller must close the ReadCloser when done reading.
func (fr *FileReader) readerForOffset(off int64) (io.ReadCloser, error) {
	if debug {
		log.Printf("(%p) readerForOffset %d + %d = %d", fr, fr.rootOff, off, fr.rootOff+off)
	}
	if off < 0 {
		panic("negative offset")
	}
	if off >= fr.size {
		return eofReader, nil
	}
	offRemain := off
	var skipped int64
	parts := fr.ss.Parts
	for len(parts) > 0 && parts[0].Size <= uint64(offRemain) {
		offRemain -= int64(parts[0].Size)
		skipped += int64(parts[0].Size)
		parts = parts[1:]
	}
	if len(parts) == 0 {
		return eofReader, nil
	}
	p0 := parts[0]
	var rsc types.ReadSeekCloser
	var err error
	switch {
	case p0.BlobRef.Valid() && p0.BytesRef.Valid():
		return nil, fmt.Errorf("part illegally contained both a blobRef and bytesRef")
	case !p0.BlobRef.Valid() && !p0.BytesRef.Valid():
		return &nZeros{int(p0.Size - uint64(offRemain))}, nil
	case p0.BlobRef.Valid():
		blob, err := blob.FromFetcher(fr.fetcher, p0.BlobRef)
		if err != nil {
			return nil, err
		}
		rsc = blob.Open()
	case p0.BytesRef.Valid():
		var ss *superset
		ss, err = fr.getSuperset(p0.BytesRef)
		if err != nil {
			return nil, err
		}
		rsc, err = ss.NewFileReader(fr.fetcher)
		if err == nil {
			subFR := rsc.(*FileReader)
			subFR.parent = fr.rootReader()
			subFR.rootOff = fr.rootOff + skipped
		}
	}
	if err != nil {
		return nil, err
	}
	offRemain += int64(p0.Offset)
	if offRemain > 0 {
		newPos, err := rsc.Seek(offRemain, os.SEEK_SET)
		if err != nil {
			return nil, err
		}
		if newPos != offRemain {
			panic("Seek didn't work")
		}
	}
	return struct {
		io.Reader
		io.Closer
	}{
		io.LimitReader(rsc, int64(p0.Size)),
		rsc,
	}, nil
}

// nZeros is a ReadCloser that reads remain zeros before EOF.
type nZeros struct {
	remain int
}

func (z *nZeros) Read(p []byte) (n int, err error) {
	for len(p) > 0 && z.remain > 0 {
		p[0] = 0
		n++
		z.remain--
	}
	if n == 0 && z.remain == 0 {
		err = io.EOF
	}
	return
}

func (*nZeros) Close() error { return nil }
<commit_msg>schema: some FileReader cleanups: use blob.Ref as map key, simplify zeroReader<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package schema

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"camlistore.org/pkg/blob"
	"camlistore.org/pkg/singleflight"
	"camlistore.org/pkg/types"
)

const closedIndex = -1

var errClosed = errors.New("filereader is closed")

// A FileReader reads the bytes of "file" and "bytes" schema blobrefs.
type FileReader struct {
	// Immutable stuff:
	*io.SectionReader             // provides Read, etc.
	parent            *FileReader // or nil for sub-region readers to find the ssm map in getSuperset
	rootOff           int64       // this FileReader's offset from the root
	fetcher           blob.Fetcher
	ss                *superset
	size              int64 // total number of bytes

	sfg singleflight.Group // for loading blobrefs for ssm

	ssmmu sync.Mutex             // guards ssm
	ssm   map[blob.Ref]*superset // blobref -> superset
}

// NewFileReader returns a new FileReader reading the contents of fileBlobRef,
// fetching blobs from fetcher.  The fileBlobRef must be of a "bytes" or "file"
// schema blob.
//
// The caller should call Close on the FileReader when done reading.
func NewFileReader(fetcher blob.Fetcher, fileBlobRef blob.Ref) (*FileReader, error) {
	// TODO(bradfitz): make this take a blobref.FetcherAt instead?
	// TODO(bradfitz): rename this into bytes reader? but for now it's still
	//                 named FileReader, but can also read a "bytes" schema.
	if !fileBlobRef.Valid() {
		return nil, errors.New("schema/filereader: NewFileReader blobref invalid")
	}
	rc, _, err := fetcher.Fetch(fileBlobRef)
	if err != nil {
		return nil, fmt.Errorf("schema/filereader: fetching file schema blob: %v", err)
	}
	defer rc.Close()
	ss, err := parseSuperset(rc)
	if err != nil {
		return nil, fmt.Errorf("schema/filereader: decoding file schema blob: %v", err)
	}
	if ss.Type != "file" && ss.Type != "bytes" {
		return nil, fmt.Errorf("schema/filereader: expected \"file\" or \"bytes\" schema blob, got %q", ss.Type)
	}
	fr, err := ss.NewFileReader(fetcher)
	if err != nil {
		return nil, fmt.Errorf("schema/filereader: creating FileReader for %s: %v", fileBlobRef, err)
	}
	return fr, nil
}

func (b *Blob) NewFileReader(fetcher blob.Fetcher) (*FileReader, error) {
	return b.ss.NewFileReader(fetcher)
}

// NewFileReader returns a new FileReader, reading bytes and blobs
// from the provided fetcher.
//
// NewFileReader does no fetch operation on the fetcher itself.  The
// fetcher is only used in subsequent read operations.
//
// An error is only returned if the type of the superset is not either
// "file" or "bytes".
func (ss *superset) NewFileReader(fetcher blob.Fetcher) (*FileReader, error) {
	if ss.Type != "file" && ss.Type != "bytes" {
		return nil, fmt.Errorf("schema/filereader: Superset not of type \"file\" or \"bytes\"")
	}
	size := int64(ss.SumPartsSize())
	fr := &FileReader{
		fetcher: fetcher,
		ss:      ss,
		size:    size,
		ssm:     make(map[blob.Ref]*superset),
	}
	fr.SectionReader = io.NewSectionReader(fr, 0, size)
	return fr, nil
}

// LoadAllChunks causes all chunks of the file to be loaded as quickly
// as possible.  The contents are immediately discarded, so it is
// assumed that the fetcher is a caching fetcher.
func (fr *FileReader) LoadAllChunks() {
	offsetc := make(chan int64, 16)
	go func() {
		for off := range offsetc {
			go func(off int64) {
				rc, err := fr.readerForOffset(off)
				if err == nil {
					defer rc.Close()
					var b [1]byte
					rc.Read(b[:]) // fault in the blob
				}
			}(off)
		}
	}()
	go fr.GetChunkOffsets(offsetc)
}

// UnixMtime returns the file schema's UnixMtime field, or the zero value.
func (fr *FileReader) UnixMtime() time.Time {
	t, err := time.Parse(time.RFC3339, fr.ss.UnixMtime)
	if err != nil {
		return time.Time{}
	}
	return t
}

// FileName returns the file schema's filename, if any.
func (fr *FileReader) FileName() string { return fr.ss.FileName }

func (fr *FileReader) Close() error {
	// TODO: close cached blobs?
	return nil
}

var _ interface {
	io.ReaderAt
	io.Reader
	io.Closer
	Size() int64
} = (*FileReader)(nil)

func (fr *FileReader) ReadAt(p []byte, offset int64) (n int, err error) {
	if offset < 0 {
		return 0, errors.New("schema/filereader: negative offset")
	}
	if offset >= fr.Size() {
		return 0, io.EOF
	}
	want := len(p)
	for len(p) > 0 && err == nil {
		var rc io.ReadCloser
		rc, err = fr.readerForOffset(offset)
		if err != nil {
			return
		}
		var n1 int64 // never bigger than an int
		n1, err = io.CopyN(&sliceWriter{p}, rc, int64(len(p)))
		rc.Close()
		if err == io.EOF {
			err = nil
		}
		if n1 == 0 {
			break
		}
		p = p[n1:]
		offset += int64(n1)
		n += int(n1)
	}
	if n < want && err == nil {
		err = io.ErrUnexpectedEOF
	}
	return n, err
}

// GetChunkOffsets sends c each of the file's chunk offsets.
// The offsets are not necessarily sent in order, and all ranges of the file
// are not necessarily represented if the file contains zero holes.
// The channel c is closed before the function returns, regardless of error.
func (fr *FileReader) GetChunkOffsets(c chan<- int64) error {
	defer close(c)
	firstErrc := make(chan error, 1)
	return fr.sendPartsChunks(c, firstErrc, 0, fr.ss.Parts)
}

// firstErrc is a communication mechanism amongst all outstanding
// superset-fetching goroutines to see if anybody else has failed.  If
// so (a non-blocking read returns something), then the recursive call
// to sendPartsChunks is skipped, hopefully preventing unnecessary
// work.  Whenever a caller receives on firstErrc, it should also send
// back to it.  It's buffered.
func (fr *FileReader) sendPartsChunks(c chan<- int64, firstErrc chan error, off int64, parts []*BytesPart) error {
	var errcs []chan error
	for _, p := range parts {
		switch {
		case p.BlobRef.Valid() && p.BytesRef.Valid():
			return fmt.Errorf("part illegally contained both a blobRef and bytesRef")
		case !p.BlobRef.Valid() && !p.BytesRef.Valid():
			// Don't send
		case p.BlobRef.Valid():
			c <- off
		case p.BytesRef.Valid():
			errc := make(chan error, 1)
			errcs = append(errcs, errc)
			br := p.BytesRef
			go func(off int64) (err error) {
				defer func() {
					errc <- err
					if err != nil {
						select {
						case firstErrc <- err: // pump
						default:
						}
					}
				}()
				select {
				case err = <-firstErrc:
					// There was already an error elsewhere in the file.
					// Avoid doing more work.
					return
				default:
					ss, err := fr.getSuperset(br)
					if err != nil {
						return err
					}
					return fr.sendPartsChunks(c, firstErrc, off, ss.Parts)
				}
			}(off)
		}
		off += int64(p.Size)
	}

	var retErr error
	for _, errc := range errcs {
		if err := <-errc; err != nil && retErr == nil {
			retErr = err
		}
	}
	return retErr
}

type sliceWriter struct {
	dst []byte
}

func (sw *sliceWriter) Write(p []byte) (n int, err error) {
	n = copy(sw.dst, p)
	sw.dst = sw.dst[n:]
	return n, nil
}

var eofReader io.ReadCloser = ioutil.NopCloser(strings.NewReader(""))

func (fr *FileReader) rootReader() *FileReader {
	if fr.parent != nil {
		return fr.parent.rootReader()
	}
	return fr
}

func (fr *FileReader) getSuperset(br blob.Ref) (*superset, error) {
	if root := fr.rootReader(); root != fr {
		return root.getSuperset(br)
	}
	brStr := br.String()
	ssi, err := fr.sfg.Do(brStr, func() (interface{}, error) {
		fr.ssmmu.Lock()
		ss, ok := fr.ssm[br]
		fr.ssmmu.Unlock()
		if ok {
			return ss, nil
		}
		rc, _, err := fr.fetcher.Fetch(br)
		if err != nil {
			return nil, fmt.Errorf("schema/filereader: fetching file schema blob: %v", err)
		}
		defer rc.Close()
		ss, err = parseSuperset(rc)
		if err != nil {
			return nil, err
		}
		fr.ssmmu.Lock()
		defer fr.ssmmu.Unlock()
		fr.ssm[br] = ss
		return ss, nil
	})
	if err != nil {
		return nil, err
	}
	return ssi.(*superset), nil
}

var debug = os.Getenv("CAMLI_DEBUG") != ""

// readerForOffset returns a ReadCloser that reads some number of bytes and then EOF
// from the provided offset.  Seeing EOF doesn't mean the end of the whole file; just the
// chunk at that offset.  The caller must close the ReadCloser when done reading.
func (fr *FileReader) readerForOffset(off int64) (io.ReadCloser, error) {
	if debug {
		log.Printf("(%p) readerForOffset %d + %d = %d", fr, fr.rootOff, off, fr.rootOff+off)
	}
	if off < 0 {
		panic("negative offset")
	}
	if off >= fr.size {
		return eofReader, nil
	}
	offRemain := off
	var skipped int64
	parts := fr.ss.Parts
	for len(parts) > 0 && parts[0].Size <= uint64(offRemain) {
		offRemain -= int64(parts[0].Size)
		skipped += int64(parts[0].Size)
		parts = parts[1:]
	}
	if len(parts) == 0 {
		return eofReader, nil
	}
	p0 := parts[0]
	var rsc types.ReadSeekCloser
	var err error
	switch {
	case p0.BlobRef.Valid() && p0.BytesRef.Valid():
		return nil, fmt.Errorf("part illegally contained both a blobRef and bytesRef")
	case !p0.BlobRef.Valid() && !p0.BytesRef.Valid():
		return ioutil.NopCloser(
			io.LimitReader(zeroReader{},
				int64(p0.Size-uint64(offRemain)))), nil
	case p0.BlobRef.Valid():
		blob, err := blob.FromFetcher(fr.fetcher, p0.BlobRef)
		if err != nil {
			return nil, err
		}
		rsc = blob.Open()
	case p0.BytesRef.Valid():
		var ss *superset
		ss, err = fr.getSuperset(p0.BytesRef)
		if err != nil {
			return nil, err
		}
		rsc, err = ss.NewFileReader(fr.fetcher)
		if err == nil {
			subFR := rsc.(*FileReader)
			subFR.parent = fr.rootReader()
			subFR.rootOff = fr.rootOff + skipped
		}
	}
	if err != nil {
		return nil, err
	}
	offRemain += int64(p0.Offset)
	if offRemain > 0 {
		newPos, err := rsc.Seek(offRemain, os.SEEK_SET)
		if err != nil {
			return nil, err
		}
		if newPos != offRemain {
			panic("Seek didn't work")
		}
	}
	return struct {
		io.Reader
		io.Closer
	}{
		io.LimitReader(rsc, int64(p0.Size)),
		rsc,
	}, nil
}

type zeroReader struct{}

func (zeroReader) Read(p []byte) (n int, err error) {
	for i := range p {
		p[i] = 0
	}
	return len(p), nil
}
<|endoftext|>


<commit_before>package shell

import (
	"testing"

	. "src.elv.sh/pkg/prog/progtest"
	"src.elv.sh/pkg/testutil"
)

func TestScript(t *testing.T) {
	setupCleanHomePaths(t)
	testutil.InTempDir(t)
	testutil.MustWriteFile("a.elv", "echo hello")

	Test(t, &Program{},
		ThatElvish("a.elv").WritesStdout("hello\n"),
		ThatElvish("-c", "echo hello").WritesStdout("hello\n"),
		ThatElvish("non-existent.elv").
			ExitsWith(2).
			WritesStderrContaining("cannot read script"),

		// parse error
		ThatElvish("-c", "echo [").
			ExitsWith(2).
			WritesStderrContaining("parse error"),
		// parse error with -compileonly
		ThatElvish("-compileonly", "-json", "-c", "echo [").
			ExitsWith(2).
			WritesStdout(`[{"fileName":"code from -c","start":6,"end":6,"message":"should be ']'"}]`+"\n"),
		// multiple parse errors with -compileonly -json
		ThatElvish("-compileonly", "-json", "-c", "echo [{").
			ExitsWith(2).
			WritesStdout(`[{"fileName":"code from -c","start":7,"end":7,"message":"should be ',' or '}'"},{"fileName":"code from -c","start":7,"end":7,"message":"should be ']'"}]`+"\n"),

		// compilation error
		ThatElvish("-c", "echo $a").
			ExitsWith(2).
			WritesStderrContaining("compilation error"),
		// compilation error with -compileonly
		ThatElvish("-compileonly", "-json", "-c", "echo $a").
			ExitsWith(2).
			WritesStdout(`[{"fileName":"code from -c","start":5,"end":7,"message":"variable $a not found"}]`+"\n"),
		// parse error and compilation error with -compileonly
		ThatElvish("-compileonly", "-json", "-c", "echo [$a").
			ExitsWith(2).
			WritesStdout(`[{"fileName":"code from -c","start":8,"end":8,"message":"should be ']'"},{"fileName":"code from -c","start":6,"end":8,"message":"variable $a not found"}]`+"\n"),

		// exception
		ThatElvish("-c", "fail failure").
			ExitsWith(2).
			WritesStdout("").
			WritesStderrContaining("fail failure"),
		// exception with -compileonly
		ThatElvish("-compileonly", "-c", "fail failure").
			ExitsWith(0),
	)
}
<commit_msg>pkg/shell: Add more scripting tests.<commit_after>package shell

import (
	"testing"

	. "src.elv.sh/pkg/prog/progtest"
	"src.elv.sh/pkg/testutil"
)

func TestScript(t *testing.T) {
	setupCleanHomePaths(t)
	testutil.InTempDir(t)
	testutil.MustWriteFile("hello.elv", "echo hello")
	testutil.MustWriteFile("invalid-utf8.elv", "\xff")

	Test(t, &Program{},
		ThatElvish("hello.elv").WritesStdout("hello\n"),
		ThatElvish("-c", "echo hello").WritesStdout("hello\n"),

		ThatElvish("invalid-utf8.elv").
			ExitsWith(2).
			WritesStderrContaining("cannot read script"),
		ThatElvish("non-existent.elv").
			ExitsWith(2).
			WritesStderrContaining("cannot read script"),

		// parse error
		ThatElvish("-c", "echo [").
			ExitsWith(2).
			WritesStderrContaining("parse error"),
		// parse error with -compileonly
		ThatElvish("-compileonly", "-c", "echo [").
			ExitsWith(2).
			WritesStderrContaining("parse error"),
		// parse error with -compileonly -json
		ThatElvish("-compileonly", "-json", "-c", "echo [").
			ExitsWith(2).
			WritesStdout(`[{"fileName":"code from -c","start":6,"end":6,"message":"should be ']'"}]`+"\n"),
		// multiple parse errors with -compileonly -json
		ThatElvish("-compileonly", "-json", "-c", "echo [{").
			ExitsWith(2).
			WritesStdout(`[{"fileName":"code from -c","start":7,"end":7,"message":"should be ',' or '}'"},{"fileName":"code from -c","start":7,"end":7,"message":"should be ']'"}]`+"\n"),

		// compilation error
		ThatElvish("-c", "echo $a").
			ExitsWith(2).
			WritesStderrContaining("compilation error"),
		// compilation error with -compileonly
		ThatElvish("-compileonly", "-c", "echo $a").
			ExitsWith(2).
			WritesStderrContaining("compilation error"),
		// compilation error with -compileonly -json
		ThatElvish("-compileonly", "-json", "-c", "echo $a").
			ExitsWith(2).
			WritesStdout(`[{"fileName":"code from -c","start":5,"end":7,"message":"variable $a not found"}]`+"\n"),
		// parse error and compilation error with -compileonly
		ThatElvish("-compileonly", "-json", "-c", "echo [$a").
			ExitsWith(2).
			WritesStdout(`[{"fileName":"code from -c","start":8,"end":8,"message":"should be ']'"},{"fileName":"code from -c","start":6,"end":8,"message":"variable $a not found"}]`+"\n"),

		// exception
		ThatElvish("-c", "fail failure").
			ExitsWith(2).
			WritesStdout("").
			WritesStderrContaining("fail failure"),
		// exception with -compileonly
		ThatElvish("-compileonly", "-c", "fail failure").
			ExitsWith(0),
	)
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package frp

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/admpub/confl"
	"github.com/webx-top/com"

	"github.com/admpub/nging/application/dbschema"

	_ "github.com/admpub/frp/assets/frpc/statik"
	_ "github.com/admpub/frp/assets/frps/statik"
	"github.com/admpub/frp/pkg/config"
	plugin "github.com/admpub/frp/pkg/plugin/server"
	frpLog "github.com/admpub/frp/pkg/util/log"
	"github.com/admpub/frp/pkg/util/util"
	"github.com/admpub/frp/server"
)

func SetServerConfigFromDB(conf *dbschema.NgingFrpServer) *config.ServerCommonConf {
	c := config.GetDefaultServerConf()
	c.BindAddr = conf.Addr
	c.BindPort = int(conf.Port)
	c.BindUDPPort = int(conf.UdpPort)
	c.KCPBindPort = int(conf.KcpPort)
	c.ProxyBindAddr = conf.ProxyAddr
	c.VhostHTTPPort = int(conf.VhostHttpPort)
	c.VhostHTTPTimeout = int64(conf.VhostHttpTimeout)
	if c.VhostHTTPTimeout < 1 {
		c.VhostHTTPTimeout = 60
	}
	c.VhostHTTPSPort = int(conf.VhostHttpsPort)

	c.DashboardAddr = conf.DashboardAddr
	c.DashboardPort = int(conf.DashboardPort)
	c.DashboardUser = conf.DashboardUser
	c.DashboardPwd = conf.DashboardPwd

	// TODO:
	//c.EnablePrometheus = false
	//c.AssetsDir = ``

	c.LogFile = conf.LogFile
	c.LogWay = conf.LogWay
	if c.LogWay == `console` {
		c.LogFile = `console`
	} else if len(c.LogFile) == 0 {
		c.LogFile = `console`
	} else {
		os.MkdirAll(filepath.Dir(c.LogFile), os.ModePerm)
	}
	c.LogLevel = conf.LogLevel
	c.LogMaxDays = int64(conf.LogMaxDays)
	c.DisableLogColor = true
	c.DetailedErrorsToClient = true
	c.Custom404Page = ``
	c.Token = conf.Token

	// TODO:
	// c.AuthenticationMethod = `token`
	// c.AuthenticateHeartBeats = false
	// c.AuthenticateNewWorkConns = false
	// c.OidcIssuer = ""
	// c.OidcAudience = ""
	// c.OidcSkipExpiryCheck = false
	// c.OidcSkipIssuerCheck = false

	c.SubDomainHost = conf.SubdomainHost
	c.MaxPoolCount = int64(conf.MaxPoolCount)
	if c.MaxPoolCount < 1 {
		c.MaxPoolCount = 5
	}
	c.MaxPortsPerClient = int64(conf.MaxPortsPerClient)
	c.TCPMux = conf.TcpMux == `Y`

	// e.g. 1000-2000,2001,2002,3000-4000
	ports, _ := util.ParseRangeNumbers(conf.AllowPorts)
	for _, port := range ports {
		c.AllowPorts[int(port)] = struct{}{}
	}

	// TODO:
	// c.TLSOnly = conf.TlsOnly
	// c.TLSCertFile = conf.TlsCertFile
	// c.TLSKeyFile = conf.TlsKeyFile
	// c.TLSTrustedCaFile = conf.TlsTrustedCaFile

	c.HeartbeatTimeout = int64(conf.HeartBeatTimeout)
	if c.HeartbeatTimeout < 1 {
		c.HeartbeatTimeout = 90
	}
	c.UserConnTimeout = int64(conf.UserConnTimeout)
	if c.UserConnTimeout < 1 {
		c.UserConnTimeout = 10
	}
	if c.HTTPPlugins == nil {
		c.HTTPPlugins = map[string]plugin.HTTPPluginOptions{}
	}
	c.UDPPacketSize = 1500
	conf.Plugins = strings.TrimSpace(conf.Plugins)
	if len(conf.Plugins) > 0 {
		plugins := strings.Split(conf.Plugins, ",")
		for name, plugin := range serverPlugins {
			if com.InSlice(name, plugins) && plugin != nil && plugin.getter != nil {
				c.HTTPPlugins[name] = plugin.getter(&c)
			}
		}
	}
	return &c
}

func StartServerByConfigFile(filePath string, pidFile string) error {
	ext := filepath.Ext(filePath)
	b, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}
	switch strings.ToLower(ext) {
	case `.json`:
		r := &dbschema.NgingFrpServer{}
		err = json.Unmarshal(b, r)
		if err != nil {
			return err
		}
		c := SetServerConfigFromDB(r)
		return StartServer(pidFile, c)
	case `.yaml`:
		r := &dbschema.NgingFrpServer{}
		err = confl.Unmarshal(b, r)
		if err != nil {
			return err
		}
		c := SetServerConfigFromDB(r)
		return StartServer(pidFile, c)
	default:
		content := string(b)
		return StartServerByConfig(content, pidFile)
	}
}

func StartServerByConfig(configContent string, pidFile string) error {
	cfg, err := config.UnmarshalServerConfFromIni(configContent)
	if err != nil {
		return err
	}
	return StartServer(pidFile, &cfg)
}

func StartServer(pidFile string, c *config.ServerCommonConf) error {
	once.Do(onceInit)
	err := c.Validate()
	if err != nil {
		return err
	}
	frpLog.InitLog(c.LogWay, c.LogFile, c.LogLevel, c.LogMaxDays, c.DisableLogColor)
	if len(pidFile) > 0 {
		err := com.WritePidFile(pidFile)
		if err != nil {
			frpLog.Error(err.Error())
			return err
		}
	}
	svr, err := server.NewService(*c)
	if err != nil {
		return err
	}
	frpLog.Info("Start frps success")
	svr.Run()
	return err
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package frp

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/admpub/confl"
	"github.com/webx-top/com"

	"github.com/admpub/nging/application/dbschema"

	_ "github.com/admpub/frp/assets/frpc/statik"
	_ "github.com/admpub/frp/assets/frps/statik"
	"github.com/admpub/frp/pkg/config"
	plugin "github.com/admpub/frp/pkg/plugin/server"
	frpLog "github.com/admpub/frp/pkg/util/log"
	"github.com/admpub/frp/pkg/util/util"
	"github.com/admpub/frp/server"
)

func SetServerConfigFromDB(conf *dbschema.NgingFrpServer) *config.ServerCommonConf {
	c := config.GetDefaultServerConf()
	c.BindAddr = conf.Addr
	c.BindPort = int(conf.Port)
	c.BindUDPPort = int(conf.UdpPort)
	c.KCPBindPort = int(conf.KcpPort)
	c.ProxyBindAddr = conf.ProxyAddr
	c.VhostHTTPPort = int(conf.VhostHttpPort)
	c.VhostHTTPTimeout = int64(conf.VhostHttpTimeout)
	if c.VhostHTTPTimeout < 1 {
		c.VhostHTTPTimeout = 60
	}
	c.VhostHTTPSPort = int(conf.VhostHttpsPort)

	c.DashboardAddr = conf.DashboardAddr
	c.DashboardPort = int(conf.DashboardPort)
	c.DashboardUser = conf.DashboardUser
	c.DashboardPwd = conf.DashboardPwd

	// TODO:
	//c.EnablePrometheus = false
	//c.AssetsDir = ``

	c.LogFile = conf.LogFile
	c.LogWay = conf.LogWay
	if c.LogWay == `console` {
		c.LogFile = `console`
	} else if len(c.LogFile) == 0 {
		c.LogFile = `console`
	} else {
		os.MkdirAll(filepath.Dir(c.LogFile), os.ModePerm)
	}
	c.LogLevel = conf.LogLevel
	c.LogMaxDays = int64(conf.LogMaxDays)
	c.DisableLogColor = true
	c.DetailedErrorsToClient = true
	c.Custom404Page = ``
	c.Token = conf.Token

	// TODO:
	// c.AuthenticationMethod = `token`
	// c.AuthenticateHeartBeats = false
	// c.AuthenticateNewWorkConns = false
	// c.OidcIssuer = ""
	// c.OidcAudience = ""
	// c.OidcSkipExpiryCheck = false
	// c.OidcSkipIssuerCheck = false

	c.SubDomainHost = conf.SubdomainHost
	c.MaxPoolCount = int64(conf.MaxPoolCount)
	if c.MaxPoolCount < 1 {
		c.MaxPoolCount = 5
	}
	c.MaxPortsPerClient = int64(conf.MaxPortsPerClient)
	c.TCPMux = conf.TcpMux == `Y`

	// e.g. 1000-2000,2001,2002,3000-4000
	ports, _ := util.ParseRangeNumbers(conf.AllowPorts)
	for _, port := range ports {
		c.AllowPorts[int(port)] = struct{}{}
	}

	// TODO:
	// c.TLSOnly = conf.TlsOnly
	// c.TLSCertFile = conf.TlsCertFile
	// c.TLSKeyFile = conf.TlsKeyFile
	// c.TLSTrustedCaFile = conf.TlsTrustedCaFile

	c.HeartbeatTimeout = int64(conf.HeartBeatTimeout)
	if c.HeartbeatTimeout < 1 {
		c.HeartbeatTimeout = 90
	}
	c.UserConnTimeout = int64(conf.UserConnTimeout)
	if c.UserConnTimeout < 1 {
		c.UserConnTimeout = 10
	}
	if c.HTTPPlugins == nil {
		c.HTTPPlugins = map[string]plugin.HTTPPluginOptions{}
	}
	c.UDPPacketSize = 1500
	conf.Plugins = strings.TrimSpace(conf.Plugins)
	if len(conf.Plugins) > 0 {
		plugins := strings.Split(conf.Plugins, ",")
		for _, name := range plugins {
			if plugin, ok := serverPlugins[name]; ok && plugin != nil && plugin.getter != nil {
				c.HTTPPlugins[name] = plugin.getter(&c)
			}
		}
	}
	return &c
}

func StartServerByConfigFile(filePath string, pidFile string) error {
	ext := filepath.Ext(filePath)
	b, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}
	switch strings.ToLower(ext) {
	case `.json`:
		r := &dbschema.NgingFrpServer{}
		err = json.Unmarshal(b, r)
		if err != nil {
			return err
		}
		c := SetServerConfigFromDB(r)
		return StartServer(pidFile, c)
	case `.yaml`:
		r := &dbschema.NgingFrpServer{}
		err = confl.Unmarshal(b, r)
		if err != nil {
			return err
		}
		c := SetServerConfigFromDB(r)
		return StartServer(pidFile, c)
	default:
		content := string(b)
		return StartServerByConfig(content, pidFile)
	}
}

func StartServerByConfig(configContent string, pidFile string) error {
	cfg, err := config.UnmarshalServerConfFromIni(configContent)
	if err != nil {
		return err
	}
	return StartServer(pidFile, &cfg)
}

func StartServer(pidFile string, c *config.ServerCommonConf) error {
	once.Do(onceInit)
	err := c.Validate()
	if err != nil {
		return err
	}
	frpLog.InitLog(c.LogWay, c.LogFile, c.LogLevel, c.LogMaxDays, c.DisableLogColor)
	if len(pidFile) > 0 {
		err := com.WritePidFile(pidFile)
		if err != nil {
			frpLog.Error(err.Error())
			return err
		}
	}
	svr, err := server.NewService(*c)
	if err != nil {
		return err
	}
	frpLog.Info("Start frps success")
	svr.Run()
	return err
}
<|endoftext|>


<commit_before>package perm

import "github.com/admpub/nging/application/library/perm"

var Behaviors = perm.NewBehaviors()

func init() {
	Behaviors.Register(`test`, `测试行为`, perm.BehaviorOptFormHelpBlock(`仅仅用于开发测试`), perm.BehaviorOptValue(`默认值`))
}
<commit_msg>update<commit_after>package perm

import "github.com/admpub/nging/application/library/perm"

var Behaviors = perm.NewBehaviors()

/*
func init() {
	Behaviors.Register(`test`, `测试行为`, perm.BehaviorOptFormHelpBlock(`仅仅用于开发测试`), perm.BehaviorOptValue(`默认值`))
}
*/
<|endoftext|>


<commit_before>package main

import (
	"crypto"
	"encoding/json"
	"github.com/realglobe-Inc/edo/util"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"net/http"
	"time"
)

const (
	jwtAlg = "alg"
	jwtKid = "kid"
)

const (
	algNone = "none"
)

const (
	grntTypeCod = "code"
)

func responseToken(w http.ResponseWriter, tok *token) error {
	m := map[string]interface{}{
		formTokId:   tok.id(),
		formTokType: tokTypeBear,
	}
	if !tok.expirationDate().IsZero() {
		m[formExpi] = int64(tok.expirationDate().Sub(time.Now()).Seconds())
	}
	if tok.refreshToken() != "" {
		m[formRefTok] = tok.refreshToken()
	}
	if len(tok.scopes()) > 0 {
		var buff string
		for scop := range tok.scopes() {
			if len(buff) > 0 {
				buff += " "
			}
			buff += scop
		}
		m[formScop] = buff
	}
	if tok.idToken() != "" {
		m[formIdTok] = tok.idToken()
	}
	buff, err := json.Marshal(m)
	if err != nil {
		return erro.Wrap(err)
	}

	w.Header().Add("Cache-Control", "no-store")
	w.Header().Add("Pragma", "no-cache")
	if _, err := w.Write(buff); err != nil {
		err = erro.Wrap(err)
		log.Err(erro.Unwrap(err))
		log.Debug(err)
	}
	return nil
}

func tokenApi(w http.ResponseWriter, r *http.Request, sys *system) error {
	if r.Method != "POST" {
		return newIdpError(errInvReq, r.Method+" is not supported", http.StatusMethodNotAllowed, nil)
	}

	req := newTokenRequest(r)
	// 重複パラメータが無いか検査。
	for k, v := range r.Form {
		if len(v) > 1 {
			return newIdpError(errInvReq, k+" is overlapped", http.StatusBadRequest, nil)
		}
	}

	if grntType := req.grantType(); grntType == "" {
		return newIdpError(errInvReq, "no "+formGrntType, http.StatusBadRequest, nil)
	} else if grntType == grntTypeCod {
		return newIdpError(errUnsuppGrntType, grntType+" is not supported", http.StatusBadRequest, nil)
	}

	log.Debug("Grant type is " + grntTypeCod)

	rawCod := req.code()
	if rawCod == "" {
		return newIdpError(errInvReq, "no "+formCod, http.StatusBadRequest, nil)
	}

	var codId string
	if codJws, err := util.ParseJws(rawCod); err != nil {
		// JWS から抜き出した ID だけ送られてきた。
		codId = rawCod
		rawCod = ""
	} else {
		// JWS のまま送られてきた。
		log.Debug("Raw code " + mosaic(rawCod) + " is declared")
		codId, _ = codJws.Claim(clmJti).(string)
	}

	log.Debug("Code " + mosaic(codId) + " is declared")

	cod, err := sys.codCont.get(codId)
	if err != nil {
		return erro.Wrap(err)
	} else if cod == nil {
		return newIdpError(errInvGrnt, "code "+mosaic(codId)+" is not exist", http.StatusBadRequest, nil)
	} else if !cod.valid() {
		// TODO 発行したアクセストークンを無効に。
		return newIdpError(errInvGrnt, "code "+mosaic(codId)+" is invalid", http.StatusBadRequest, nil)
	}

	log.Debug("Code " + mosaic(codId) + " is exist")

	// 認可コードを使用済みにする。
	cod.disable()
	if err := sys.codCont.put(cod); err != nil {
		return newIdpError(errServErr, erro.Unwrap(err).Error(), http.StatusBadRequest, erro.Wrap(err))
	}

	log.Debug("Code " + mosaic(codId) + " is disabled")

	taId := req.taId()
	if taId == "" {
		taId = cod.taId()
		log.Debug("TA ID is " + taId + " in code")
	} else if taId != cod.taId() {
		return newIdpError(errInvTa, "you are not code holder", http.StatusBadRequest, nil)
	} else {
		log.Debug("TA ID " + taId + " is declared")
	}

	rediUri := req.redirectUri()
	if rediUri == "" {
		return newIdpError(errInvReq, "no "+formRediUri, http.StatusBadRequest, nil)
	} else if rediUri != cod.redirectUri() {
		return newIdpError(errInvTa, "invalid "+formRediUri, http.StatusBadRequest, nil)
	}

	log.Debug(formRediUri + " matches that of code")

	if taAssType := req.taAssertionType(); taAssType == "" {
		return newIdpError(errInvTa, "no "+formTaAssType, http.StatusBadRequest, nil)
	} else if taAssType != taAssTypeJwt {
		return newIdpError(errInvTa, taAssType+" is not supported", http.StatusBadRequest, nil)
	}

	log.Debug(formTaAssType + " is " + taAssTypeJwt)

	taAss := req.taAssertion()
	if taAss == "" {
		return newIdpError(errInvTa, "no "+formTaAss, http.StatusBadRequest, nil)
	}

	log.Debug(formTaAss + " is found")

	// クライアント認証する。
	assJws, err := util.ParseJws(taAss)
	if err != nil {
		err = erro.Wrap(err)
		log.Err(erro.Unwrap(err))
		log.Debug(err)
		return newIdpError(errInvTa, erro.Unwrap(err).Error(), http.StatusBadRequest, nil)
	}

	now := time.Now()
	if assJws.Claim(clmIss) != taId {
		return newIdpError(errInvTa, "assertion "+clmIss+" is not "+taId, http.StatusBadRequest, nil)
	} else if assJws.Claim(clmSub) != taId {
		return newIdpError(errInvTa, "assertion "+clmSub+" is not "+taId, http.StatusBadRequest, nil)
	} else if jti := assJws.Claim(clmJti); jti == nil || jti == "" {
		return newIdpError(errInvTa, "no assertion "+clmJti, http.StatusBadRequest, nil)
	} else if exp, _ := assJws.Claim(clmExp).(float64); exp == 0 {
		return newIdpError(errInvTa, "no assertion "+clmExp, http.StatusBadRequest, nil)
	} else if intExp := int64(exp); exp != float64(intExp) {
		return newIdpError(errInvTa, "assertion "+clmExp+" is not integer", http.StatusBadRequest, nil)
	} else if intExp < now.Unix() {
		return newIdpError(errInvTa, "assertion expired", http.StatusBadRequest, nil)
	} else if aud := assJws.Claim(clmAud); aud == nil {
		return newIdpError(errInvTa, "no assertion "+clmAud, http.StatusBadRequest, nil)
	} else if !audienceHas(aud, sys.selfId+tokPath) {
		return newIdpError(errInvTa, "assertion "+clmAud+" does not contain "+sys.selfId+tokPath, http.StatusBadRequest, nil)
	} else if c := assJws.Claim(clmCod); !((rawCod != "" || c == rawCod) || c == codId) {
		return newIdpError(errInvTa, "invalid assertion "+clmCod, http.StatusBadRequest, nil)
	}

	// クライアント認証情報は揃ってた。
	log.Debug("Assertion claims are OK")

	ta, err := sys.taCont.get(taId)
	if err != nil {
		return erro.Wrap(err)
	}

	if assJws.Header(jwtAlg) == algNone {
		return newIdpError(errInvTa, "asserion "+jwtAlg+" must not be "+algNone, http.StatusBadRequest, nil)
	} else if err := assJws.Verify(ta.keys()); err != nil {
		return newIdpError(errInvTa, erro.Unwrap(err).Error(), http.StatusBadRequest, erro.Wrap(err))
	}

	// クライアント認証できた。
	log.Debug(taId + " is authenticated")

	idTokJws := util.NewJws()
	idTokJws.SetHeader(jwtAlg, sys.sigAlg)
	if sys.sigKid != "" {
		idTokJws.SetHeader(jwtKid, sys.sigKid)
	}
	idTokJws.SetClaim(clmIss, sys.selfId)
	idTokJws.SetClaim(clmSub, cod.accountId())
	idTokJws.SetClaim(clmAud, cod.taId())
	idTokJws.SetClaim(clmExp, now.Add(sys.idTokExpiDur).Unix())
	idTokJws.SetClaim(clmIat, now.Unix())
	if !cod.authenticationDate().IsZero() {
		idTokJws.SetClaim(clmAuthTim, cod.authenticationDate().Unix())
	}
	if cod.nonce() != "" {
		idTokJws.SetClaim(clmNonc, cod.nonce())
	}
	if err := idTokJws.Sign(map[string]crypto.PrivateKey{sys.sigKid: sys.sigKey}); err != nil {
		return erro.Wrap(err)
	}
	buff, err := idTokJws.Encode()
	if err != nil {
		return erro.Wrap(err)
	}
	idTok := string(buff)

	// ID トークンができた。
	log.Debug("ID token was generated")

	tokId, err := sys.tokCont.newId()
	if err != nil {
		return erro.Wrap(err)
	}
	tok := newToken(
		tokId,
		cod.accountId(),
		cod.taId(),
		cod.id(),
		"",
		now.Add(cod.expirationDuration()),
		cod.scopes(),
		cod.claims(),
		idTok,
	)
	if err := sys.tokCont.put(tok); err != nil {
		return erro.Wrap(err)
	}

	// アクセストークンが決まった。
	log.Debug("Token " + mosaic(tok.id()) + " is generated")

	return responseToken(w, tok)
}

// aud クレーム値が tgt を含むかどうか検査。
func audienceHas(aud interface{}, tgt string) bool {
	switch a := aud.(type) {
	case string:
		return a == tgt
	case []interface{}:
		for _, elem := range a {
			s, _ := elem.(string)
			if s == tgt {
				return true
			}
		}
		return false
	default:
		return false
	}
}
<commit_msg>トークンリクエストの grant_type が code で client_id が無い場合は拒否するように修正<commit_after>package main

import (
	"crypto"
	"encoding/json"
	"github.com/realglobe-Inc/edo/util"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"net/http"
	"time"
)

const (
	jwtAlg = "alg"
	jwtKid = "kid"
)

const (
	algNone = "none"
)

const (
	grntTypeCod = "code"
)

func responseToken(w http.ResponseWriter, tok *token) error {
	m := map[string]interface{}{
		formTokId:   tok.id(),
		formTokType: tokTypeBear,
	}
	if !tok.expirationDate().IsZero() {
		m[formExpi] = int64(tok.expirationDate().Sub(time.Now()).Seconds())
	}
	if tok.refreshToken() != "" {
		m[formRefTok] = tok.refreshToken()
	}
	if len(tok.scopes()) > 0 {
		var buff string
		for scop := range tok.scopes() {
			if len(buff) > 0 {
				buff += " "
			}
			buff += scop
		}
		m[formScop] = buff
	}
	if tok.idToken() != "" {
		m[formIdTok] = tok.idToken()
	}
	buff, err := json.Marshal(m)
	if err != nil {
		return erro.Wrap(err)
	}

	w.Header().Add("Cache-Control", "no-store")
	w.Header().Add("Pragma", "no-cache")
	if _, err := w.Write(buff); err != nil {
		err = erro.Wrap(err)
		log.Err(erro.Unwrap(err))
		log.Debug(err)
	}
	return nil
}

func tokenApi(w http.ResponseWriter, r *http.Request, sys *system) error {
	if r.Method != "POST" {
		return newIdpError(errInvReq, r.Method+" is not supported", http.StatusMethodNotAllowed, nil)
	}

	req := newTokenRequest(r)
	// 重複パラメータが無いか検査。
	for k, v := range r.Form {
		if len(v) > 1 {
			return newIdpError(errInvReq, k+" is overlapped", http.StatusBadRequest, nil)
		}
	}

	if grntType := req.grantType(); grntType == "" {
		return newIdpError(errInvReq, "no "+formGrntType, http.StatusBadRequest, nil)
	} else if grntType == grntTypeCod {
		return newIdpError(errUnsuppGrntType, grntType+" is not supported", http.StatusBadRequest, nil)
	}

	log.Debug("Grant type is " + grntTypeCod)

	rawCod := req.code()
	if rawCod == "" {
		return newIdpError(errInvReq, "no "+formCod, http.StatusBadRequest, nil)
	}

	var codId string
	if codJws, err := util.ParseJws(rawCod); err != nil {
		// JWS から抜き出した ID だけ送られてきた。
		codId = rawCod
		rawCod = ""
	} else {
		// JWS のまま送られてきた。
		log.Debug("Raw code " + mosaic(rawCod) + " is declared")
		codId, _ = codJws.Claim(clmJti).(string)
	}

	log.Debug("Code " + mosaic(codId) + " is declared")

	cod, err := sys.codCont.get(codId)
	if err != nil {
		return erro.Wrap(err)
	} else if cod == nil {
		return newIdpError(errInvGrnt, "code "+mosaic(codId)+" is not exist", http.StatusBadRequest, nil)
	} else if !cod.valid() {
		// TODO 発行したアクセストークンを無効に。
		return newIdpError(errInvGrnt, "code "+mosaic(codId)+" is invalid", http.StatusBadRequest, nil)
	}

	log.Debug("Code " + mosaic(codId) + " is exist")

	// 認可コードを使用済みにする。
	cod.disable()
	if err := sys.codCont.put(cod); err != nil {
		return newIdpError(errServErr, erro.Unwrap(err).Error(), http.StatusBadRequest, erro.Wrap(err))
	}

	log.Debug("Code " + mosaic(codId) + " is disabled")

	taId := req.taId()
	if taId == "" {
		return newIdpError(errInvReq, "no "+formTaId, http.StatusBadRequest, nil)
	} else if taId != cod.taId() {
		return newIdpError(errInvTa, "you are not code holder", http.StatusBadRequest, nil)
	} else {
		log.Debug("TA ID " + taId + " is declared")
	}

	rediUri := req.redirectUri()
	if rediUri == "" {
		return newIdpError(errInvReq, "no "+formRediUri, http.StatusBadRequest, nil)
	} else if rediUri != cod.redirectUri() {
		return newIdpError(errInvTa, "invalid "+formRediUri, http.StatusBadRequest, nil)
	}

	log.Debug(formRediUri + " matches that of code")

	if taAssType := req.taAssertionType(); taAssType == "" {
		return newIdpError(errInvTa, "no "+formTaAssType, http.StatusBadRequest, nil)
	} else if taAssType != taAssTypeJwt {
		return newIdpError(errInvTa, taAssType+" is not supported", http.StatusBadRequest, nil)
	}

	log.Debug(formTaAssType + " is " + taAssTypeJwt)

	taAss := req.taAssertion()
	if taAss == "" {
		return newIdpError(errInvTa, "no "+formTaAss, http.StatusBadRequest, nil)
	}

	log.Debug(formTaAss + " is found")

	// クライアント認証する。
	assJws, err := util.ParseJws(taAss)
	if err != nil {
		err = erro.Wrap(err)
		log.Err(erro.Unwrap(err))
		log.Debug(err)
		return newIdpError(errInvTa, erro.Unwrap(err).Error(), http.StatusBadRequest, nil)
	}

	now := time.Now()
	if assJws.Claim(clmIss) != taId {
		return newIdpError(errInvTa, "assertion "+clmIss+" is not "+taId, http.StatusBadRequest, nil)
	} else if assJws.Claim(clmSub) != taId {
		return newIdpError(errInvTa, "assertion "+clmSub+" is not "+taId, http.StatusBadRequest, nil)
	} else if jti := assJws.Claim(clmJti); jti == nil || jti == "" {
		return newIdpError(errInvTa, "no assertion "+clmJti, http.StatusBadRequest, nil)
	} else if exp, _ := assJws.Claim(clmExp).(float64); exp == 0 {
		return newIdpError(errInvTa, "no assertion "+clmExp, http.StatusBadRequest, nil)
	} else if intExp := int64(exp); exp != float64(intExp) {
		return newIdpError(errInvTa, "assertion "+clmExp+" is not integer", http.StatusBadRequest, nil)
	} else if intExp < now.Unix() {
		return newIdpError(errInvTa, "assertion expired", http.StatusBadRequest, nil)
	} else if aud := assJws.Claim(clmAud); aud == nil {
		return newIdpError(errInvTa, "no assertion "+clmAud, http.StatusBadRequest, nil)
	} else if !audienceHas(aud, sys.selfId+tokPath) {
		return newIdpError(errInvTa, "assertion "+clmAud+" does not contain "+sys.selfId+tokPath, http.StatusBadRequest, nil)
	} else if c := assJws.Claim(clmCod); !((rawCod != "" || c == rawCod) || c == codId) {
		return newIdpError(errInvTa, "invalid assertion "+clmCod, http.StatusBadRequest, nil)
	}

	// クライアント認証情報は揃ってた。
	log.Debug("Assertion claims are OK")

	ta, err := sys.taCont.get(taId)
	if err != nil {
		return erro.Wrap(err)
	}

	if assJws.Header(jwtAlg) == algNone {
		return newIdpError(errInvTa, "asserion "+jwtAlg+" must not be "+algNone, http.StatusBadRequest, nil)
	} else if err := assJws.Verify(ta.keys()); err != nil {
		return newIdpError(errInvTa, erro.Unwrap(err).Error(), http.StatusBadRequest, erro.Wrap(err))
	}

	// クライアント認証できた。
	log.Debug(taId + " is authenticated")

	idTokJws := util.NewJws()
	idTokJws.SetHeader(jwtAlg, sys.sigAlg)
	if sys.sigKid != "" {
		idTokJws.SetHeader(jwtKid, sys.sigKid)
	}
	idTokJws.SetClaim(clmIss, sys.selfId)
	idTokJws.SetClaim(clmSub, cod.accountId())
	idTokJws.SetClaim(clmAud, cod.taId())
	idTokJws.SetClaim(clmExp, now.Add(sys.idTokExpiDur).Unix())
	idTokJws.SetClaim(clmIat, now.Unix())
	if !cod.authenticationDate().IsZero() {
		idTokJws.SetClaim(clmAuthTim, cod.authenticationDate().Unix())
	}
	if cod.nonce() != "" {
		idTokJws.SetClaim(clmNonc, cod.nonce())
	}
	if err := idTokJws.Sign(map[string]crypto.PrivateKey{sys.sigKid: sys.sigKey}); err != nil {
		return erro.Wrap(err)
	}
	buff, err := idTokJws.Encode()
	if err != nil {
		return erro.Wrap(err)
	}
	idTok := string(buff)

	// ID トークンができた。
	log.Debug("ID token was generated")

	tokId, err := sys.tokCont.newId()
	if err != nil {
		return erro.Wrap(err)
	}
	tok := newToken(
		tokId,
		cod.accountId(),
		cod.taId(),
		cod.id(),
		"",
		now.Add(cod.expirationDuration()),
		cod.scopes(),
		cod.claims(),
		idTok,
	)
	if err := sys.tokCont.put(tok); err != nil {
		return erro.Wrap(err)
	}

	// アクセストークンが決まった。
	log.Debug("Token " + mosaic(tok.id()) + " is generated")

	return responseToken(w, tok)
}

// aud クレーム値が tgt を含むかどうか検査。
func audienceHas(aud interface{}, tgt string) bool {
	switch a := aud.(type) {
	case string:
		return a == tgt
	case []interface{}:
		for _, elem := range a {
			s, _ := elem.(string)
			if s == tgt {
				return true
			}
		}
		return false
	default:
		return false
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"encoding/binary"
	"errors"
	"hash/crc32"
	"io"
	"os"
	"runtime"
	"sync"
	"sync/atomic"
)

var (
	// ErrIsDirectory means that directory is found where file assumed.
	ErrIsDirectory = errors.New("Got directory, need file")
	// ErrBadHeaderMagic means that blob is unable to initialize due header corruption or wrong file format.
	ErrBadHeader = errors.New("Bad header magic bytes, file corrupted or in wrong format")
	// ErrBadHeaderCapacity means that decoded capacity from header is less than actual file size.
	ErrBadHeaderCapacity = errors.New("Capacity in header is less than actual file size, file can be corrupted")
	// ErrBadHeaderCRC means that header crc check failed.
	ErrBadHeaderCRC = errors.New("Header CRC missmatch")
)

// Allocator wraps Allocate method for allocating slices. Should be goroutine-safe.
type Allocator interface {
	Allocate(size int64) (offset int64, err error)
}

// StatBackend wraps Stat method for retrieving file stats.
type StatBackend interface {
	Stat() (os.FileInfo, error)
}

// TruncateSyncer is the interface that groups Sync and Truncate methods.
type TruncateSyncer interface {
	// Sync commits the current contents of the file to stable storage.
	Sync() error
	// Truncate changes the size of the storage.
	Truncate(size int64) error
}

// BlobBackend is the interface that groups basic methods for consistent storage.
type BlobBackend interface {
	io.ReaderAt
	io.WriterAt
	io.Closer
	TruncateSyncer
}

// Blob represents set of data slices on top of BlobBackend.
type Blob struct {
	sync.RWMutex
	Backend    BlobBackend
	Size       int64
	Capacity   int64
	headerBuff [blobHeaderSize]byte
}

// Sync commits the current state of blob.
func (b *Blob) Sync() (err error) {
	b.Lock()
	if err = b.writeHeader(); err == nil {
		err = b.Backend.Sync()
	}
	b.Unlock()
	return err
}

// Truncate changes the capacity of the blob.
func (b *Blob) Truncate(size int64) (err error) {
	b.Lock()
	if err = b.Backend.Truncate(size); err == nil {
		b.Capacity = size
		// rendering capacity changes to header
		err = b.writeHeader()
	}
	b.Unlock()
	return err
}

// Allocate returns offset to atomically allocated slice of provided size and error if any.
// After allocation it is safe to call WriteAt(b, offset) with len(b) = size.
//
// Example:
//     data := make([]byte, size)
//     offset, _ := b.Allocate(size)
//     b.WriteAt(data, offset)
func (b *Blob) Allocate(size int64) (int64, error) {
	newSize := atomic.AddInt64(&b.Size, size)
	return newSize - size, nil
}

// readHeader encodes header to start of backend and returns error if any.
// It is not goroutine-safe. Can return errors from backend.
// Uses b.headerBuff as write buffer.
func (b *Blob) writeHeader() error {
	header := BlobHeader{
		Size:     b.Size,
		Capacity: b.Capacity,
	}
	header.Put(b.headerBuff[:])
	_, err := b.Backend.WriteAt(b.headerBuff[:], 0)
	if b.Size == 0 {
		b.Size = HeaderStructureSize
	}
	return err
}

// readHeader decodes header from start of backend and returns error if any.
// It is not goroutine-safe.
// Can return ErrBadHeaderCapacity, ErrBadHeader and errors from backend.
// Uses b.headerBuff as read buffer.
func (b *Blob) readHeader() error {
	if _, err := b.Backend.ReadAt(b.headerBuff[:], 0); err != nil {
		return err
	}
	h := BlobHeader{}
	if err := h.Read(b.headerBuff[:]); err != nil {
		return err
	}
	if b.Capacity < h.Capacity {
		return ErrBadHeaderCapacity
	}
	b.Size = h.Size
	return nil
}

func NewFileBlob(backend BlobBackend) (*Blob, error) {
	statBackend := backend.(StatBackend)
	info, err := statBackend.Stat()
	if err != nil {
		return nil, err
	}
	return &Blob{
		Size:     0,
		Capacity: info.Size(),
		Backend:  backend,
	}, nil
}

const (
	// DefaultBlobSize is initial capacity for newly created blob.
	DefaultBlobSize = 1024
)

// BlobConfig is configuration for blob processing.
type BlobConfig struct {
	InitialSize int64
}

// GetInitialSize returns initial size for the blob used upon creation.
func (i *BlobConfig) GetInitialSize() int64 {
	if i == nil || i.InitialSize == 0 {
		return DefaultBlobSize
	}
	return i.InitialSize
}

// OpenBlob opens or creates a Blob for the given path.
//
// The returned Blob instance is goroutine-safe.
// The Blob must be closed after use, by calling Close method.
func OpenBlob(path string, cfg *BlobConfig) (*Blob, error) {
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return nil, err
	}
	stat, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if stat.IsDir() {
		return nil, ErrIsDirectory
	}
	b := &Blob{
		Capacity: stat.Size(),
		Size:     0,
		Backend:  f,
	}
	if b.Capacity == 0 {
		err = b.Truncate(cfg.GetInitialSize())
	} else {
		err = b.readHeader()
	}
	runtime.SetFinalizer(b, (*Blob).Close)
	return b, err
}

// Close closes the Blob, rendering it unusable for changes.
// It returns an error, if any.
func (b Blob) Close() error {
	if err := b.Sync(); err != nil {
		return err
	}
	return b.Backend.Close()
}

// BlobHeader contains info about Blob size and capacity.
type BlobHeader struct {
	Size     int64
	Capacity int64
}

// blobHeaderSize = magic + size + capacity + crc
const blobHeaderSize = 8 + 8 + 8 + 8

var blobHeaderMagic = [...]byte{
	0xbb,
	0xba,
	0xbd,
	0xbb,
	0x13,
	0x37,
	0x20,
	0x16,
}

// Put encodes BlobHeader into buf and returns the number of bytes written.
// If the buffer is too small, Put will panic.
func (h BlobHeader) Put(buf []byte) int {
	var offset = 8
	copy(buf[:offset], blobHeaderMagic[:])
	binary.PutVarint(buf[offset:], h.Size)
	offset += 8
	binary.PutVarint(buf[offset:offset+8], h.Capacity)
	offset += 8
	crc := crc32.ChecksumIEEE(buf[:offset])
	binary.PutUvarint(buf[offset:offset+8], uint64(crc))
	offset += 8
	return offset
}

// Read decodes BlobHeader from buf and returns ErrBadHeader if it fails.
func (h *BlobHeader) Read(buf []byte) error {
	var (
		offset = 8
		crc    uint64
	)
	// checking header magic
	for i, v := range blobHeaderMagic {
		if v != buf[i] {
			return ErrBadHeader
		}
	}
	h.Size, _ = binary.Varint(buf[offset:])
	offset += 8
	h.Capacity, _ = binary.Varint(buf[offset:])
	offset += 8
	expectedCRC := crc32.ChecksumIEEE(buf[:offset])
	crc, _ = binary.Uvarint(buf[offset:])
	if crc != uint64(expectedCRC) {
		return ErrBadHeaderCRC
	}
	return nil
}

<commit_msg>refactor blob Read<commit_after>package storage

import (
	"encoding/binary"
	"errors"
	"hash/crc32"
	"io"
	"os"
	"runtime"
	"sync"
	"sync/atomic"
)

var (
	// ErrIsDirectory means that directory is found where file assumed.
	ErrIsDirectory = errors.New("Got directory, need file")
	// ErrBadHeaderMagic means that blob is unable to initialize due header corruption or wrong file format.
	ErrBadHeader = errors.New("Bad header magic bytes, file corrupted or in wrong format")
	// ErrBadHeaderCapacity means that decoded capacity from header is less than actual file size.
	ErrBadHeaderCapacity = errors.New("Capacity in header is less than actual file size, file can be corrupted")
	// ErrBadHeaderCRC means that header crc check failed.
	ErrBadHeaderCRC = errors.New("Header CRC missmatch")
)

// Allocator wraps Allocate method for allocating slices. Should be goroutine-safe.
type Allocator interface {
	Allocate(size int64) (offset int64, err error)
}

// StatBackend wraps Stat method for retrieving file stats.
type StatBackend interface {
	Stat() (os.FileInfo, error)
}

// TruncateSyncer is the interface that groups Sync and Truncate methods.
type TruncateSyncer interface {
	// Sync commits the current contents of the file to stable storage.
	Sync() error
	// Truncate changes the size of the storage.
	Truncate(size int64) error
}

// BlobBackend is the interface that groups basic methods for consistent storage.
type BlobBackend interface {
	io.ReaderAt
	io.WriterAt
	io.Closer
	TruncateSyncer
}

// Blob represents set of data slices on top of BlobBackend.
type Blob struct {
	sync.RWMutex
	Backend    BlobBackend
	Size       int64
	Capacity   int64
	headerBuff [blobHeaderSize]byte
}

// Sync commits the current state of blob.
func (b *Blob) Sync() (err error) {
	b.Lock()
	if err = b.writeHeader(); err == nil {
		err = b.Backend.Sync()
	}
	b.Unlock()
	return err
}

// Truncate changes the capacity of the blob.
func (b *Blob) Truncate(size int64) (err error) {
	b.Lock()
	if err = b.Backend.Truncate(size); err == nil {
		b.Capacity = size
		// rendering capacity changes to header
		err = b.writeHeader()
	}
	b.Unlock()
	return err
}

// Allocate returns offset to atomically allocated slice of provided size and error if any.
// After allocation it is safe to call WriteAt(b, offset) with len(b) = size.
//
// Example:
//     data := make([]byte, size)
//     offset, _ := b.Allocate(size)
//     b.WriteAt(data, offset)
func (b *Blob) Allocate(size int64) (int64, error) {
	newSize := atomic.AddInt64(&b.Size, size)
	return newSize - size, nil
}

// readHeader encodes header to start of backend and returns error if any.
// It is not goroutine-safe. Can return errors from backend.
// Uses b.headerBuff as write buffer.
func (b *Blob) writeHeader() error {
	header := BlobHeader{
		Size:     b.Size,
		Capacity: b.Capacity,
	}
	header.Put(b.headerBuff[:])
	_, err := b.Backend.WriteAt(b.headerBuff[:], 0)
	if b.Size == 0 {
		b.Size = HeaderStructureSize
	}
	return err
}

// readHeader decodes header from start of backend and returns error if any.
// It is not goroutine-safe.
// Can return ErrBadHeaderCapacity, ErrBadHeader and errors from backend.
// Uses b.headerBuff as read buffer.
func (b *Blob) readHeader() error {
	if _, err := b.Backend.ReadAt(b.headerBuff[:], 0); err != nil {
		return err
	}
	h := BlobHeader{}
	if err := h.Read(b.headerBuff[:]); err != nil {
		return err
	}
	if b.Capacity < h.Capacity {
		return ErrBadHeaderCapacity
	}
	b.Size = h.Size
	return nil
}

func NewFileBlob(backend BlobBackend) (*Blob, error) {
	statBackend := backend.(StatBackend)
	info, err := statBackend.Stat()
	if err != nil {
		return nil, err
	}
	return &Blob{
		Size:     0,
		Capacity: info.Size(),
		Backend:  backend,
	}, nil
}

const (
	// DefaultBlobSize is initial capacity for newly created blob.
	DefaultBlobSize = 1024
)

// BlobConfig is configuration for blob processing.
type BlobConfig struct {
	InitialSize int64
}

// GetInitialSize returns initial size for the blob used upon creation.
func (i *BlobConfig) GetInitialSize() int64 {
	if i == nil || i.InitialSize == 0 {
		return DefaultBlobSize
	}
	return i.InitialSize
}

// OpenBlob opens or creates a Blob for the given path.
//
// The returned Blob instance is goroutine-safe.
// The Blob must be closed after use, by calling Close method.
func OpenBlob(path string, cfg *BlobConfig) (*Blob, error) {
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return nil, err
	}
	stat, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if stat.IsDir() {
		return nil, ErrIsDirectory
	}
	b := &Blob{
		Capacity: stat.Size(),
		Size:     0,
		Backend:  f,
	}
	if b.Capacity == 0 {
		err = b.Truncate(cfg.GetInitialSize())
	} else {
		err = b.readHeader()
	}
	runtime.SetFinalizer(b, (*Blob).Close)
	return b, err
}

// Close closes the Blob, rendering it unusable for changes.
// It returns an error, if any.
func (b Blob) Close() error {
	if err := b.Sync(); err != nil {
		return err
	}
	return b.Backend.Close()
}

// BlobHeader contains info about Blob size and capacity.
type BlobHeader struct {
	Size     int64
	Capacity int64
}

// blobHeaderSize = magic + size + capacity + crc
const blobHeaderSize = 8 + 8 + 8 + 8

var blobHeaderMagic = [...]byte{
	0xbb,
	0xba,
	0xbd,
	0xbb,
	0x13,
	0x37,
	0x20,
	0x16,
}

// Put encodes BlobHeader into buf and returns the number of bytes written.
// If the buffer is too small, Put will panic.
func (h BlobHeader) Put(buf []byte) int {
	var offset = 8
	copy(buf[:offset], blobHeaderMagic[:])
	binary.PutVarint(buf[offset:], h.Size)
	offset += 8
	binary.PutVarint(buf[offset:offset+8], h.Capacity)
	offset += 8
	crc := crc32.ChecksumIEEE(buf[:offset])
	binary.PutUvarint(buf[offset:offset+8], uint64(crc))
	offset += 8
	return offset
}

// Read decodes BlobHeader from buf and returns ErrBadHeader if it fails.
func (h *BlobHeader) Read(buf []byte) error {
	// checking header magic
	for i, v := range blobHeaderMagic {
		if v != buf[i] {
			return ErrBadHeader
		}
	}
	offset := 8
	h.Size, _ = binary.Varint(buf[offset:])
	offset += 8
	h.Capacity, _ = binary.Varint(buf[offset:])
	offset += 8
	// calculating crc
	expectedCRC := crc32.ChecksumIEEE(buf[:offset])
	// checking crc
	if crc, _ := binary.Uvarint(buf[offset:]); crc != uint64(expectedCRC) {
		return ErrBadHeaderCRC
	}
	return nil
}

<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"errors"
	"fmt"

	"golang.org/x/net/context"
	raw "google.golang.org/api/storage/v1"
)

// CopierFrom creates a Copier that can copy src to dst.
// You can immediately call Run on the returned Copier, or
// you can configure it first.
func (dst *ObjectHandle) CopierFrom(src *ObjectHandle) *Copier {
	return &Copier{dst: dst, src: src}
}

// A Copier copies a source object to a destination.
type Copier struct {
	// ObjectAttrs are optional attributes to set on the destination object.
	// Any attributes must be initialized before any calls on the Copier. Nil
	// or zero-valued attributes are ignored.
	ObjectAttrs

	// RewriteToken can be set before calling Run to resume a copy
	// operation. After Run returns a non-nil error, RewriteToken will
	// have been updated to contain the value needed to resume the copy.
	RewriteToken string

	// ProgressFunc can be used to monitor the progress of a multi-RPC copy
	// operation. If ProgressFunc is not nil and CopyFrom requires multiple
	// calls to the underlying service (see
	// https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite), then
	// ProgressFunc will be invoked after each call with the number of bytes of
	// content copied so far and the total size in bytes of the source object.
	//
	// ProgressFunc is intended to make upload progress available to the
	// application. For example, the implementation of ProgressFunc may update
	// a progress bar in the application's UI, or log the result of
	// float64(copiedBytes)/float64(totalBytes).
	//
	// ProgressFunc should return quickly without blocking.
	ProgressFunc func(copiedBytes, totalBytes uint64)

	dst, src *ObjectHandle
}

// Run performs the copy.
func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) {
	if err := c.src.validate(); err != nil {
		return nil, err
	}
	if err := c.dst.validate(); err != nil {
		return nil, err
	}
	// Convert destination attributes to raw form, omitting the bucket.
	// If the bucket is included but name or content-type aren't, the service
	// returns a 400 with "Required" as the only message. Omitting the bucket
	// does not cause any problems.
	rawObject := c.ObjectAttrs.toRawObject("")
	for {
		res, err := c.callRewrite(ctx, c.src, rawObject)
		if err != nil {
			return nil, err
		}
		if c.ProgressFunc != nil {
			c.ProgressFunc(res.TotalBytesRewritten, res.ObjectSize)
		}
		if res.Done { // Finished successfully.
			return newObject(res.Resource), nil
		}
	}
	return nil, nil
}

func (c *Copier) callRewrite(ctx context.Context, src *ObjectHandle, rawObj *raw.Object) (*raw.RewriteResponse, error) {
	call := c.dst.c.raw.Objects.Rewrite(src.bucket, src.object, c.dst.bucket, c.dst.object, rawObj)

	call.Context(ctx).Projection("full")
	if c.RewriteToken != "" {
		call.RewriteToken(c.RewriteToken)
	}
	if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil {
		return nil, err
	}
	if err := applySourceConds(c.src.gen, c.src.conds, call); err != nil {
		return nil, err
	}
	if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
		return nil, err
	}
	if err := setEncryptionHeaders(call.Header(), c.src.encryptionKey, true); err != nil {
		return nil, err
	}
	var res *raw.RewriteResponse
	var err error
	setClientHeader(call.Header())
	err = runWithRetry(ctx, func() error { res, err = call.Do(); return err })
	if err != nil {
		return nil, err
	}
	c.RewriteToken = res.RewriteToken
	return res, nil
}

// ComposerFrom creates a Composer that can compose srcs into dst.
// You can immediately call Run on the returned Composer, or you can
// configure it first.
//
// The encryption key for the destination object will be used to decrypt all
// source objects and encrypt the destination object. It is an error
// to specify an encryption key for any of the source objects.
func (dst *ObjectHandle) ComposerFrom(srcs ...*ObjectHandle) *Composer {
	return &Composer{dst: dst, srcs: srcs}
}

// A Composer composes source objects into a destination object.
type Composer struct {
	// ObjectAttrs are optional attributes to set on the destination object.
	// Any attributes must be initialized before any calls on the Composer. Nil
	// or zero-valued attributes are ignored.
	ObjectAttrs

	dst  *ObjectHandle
	srcs []*ObjectHandle
}

// Run performs the compose operation.
func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) {
	if err := c.dst.validate(); err != nil {
		return nil, err
	}
	if len(c.srcs) == 0 {
		return nil, errors.New("storage: at least one source object must be specified")
	}

	req := &raw.ComposeRequest{}
	// Compose requires a non-empty Destination, so we always set it,
	// even if the caller-provided ObjectAttrs is the zero value.
	req.Destination = c.ObjectAttrs.toRawObject(c.dst.bucket)
	for _, src := range c.srcs {
		if err := src.validate(); err != nil {
			return nil, err
		}
		if src.bucket != c.dst.bucket {
			return nil, fmt.Errorf("storage: all source objects must be in bucket %q, found %q", c.dst.bucket, src.bucket)
		}
		if src.encryptionKey != nil {
			return nil, fmt.Errorf("storage: compose source %s.%s must not have encryption key", src.bucket, src.object)
		}
		srcObj := &raw.ComposeRequestSourceObjects{
			Name: src.object,
		}
		if err := applyConds("ComposeFrom source", src.gen, src.conds, composeSourceObj{srcObj}); err != nil {
			return nil, err
		}
		req.SourceObjects = append(req.SourceObjects, srcObj)
	}

	call := c.dst.c.raw.Objects.Compose(c.dst.bucket, c.dst.object, req).Context(ctx)
	if err := applyConds("ComposeFrom destination", c.dst.gen, c.dst.conds, call); err != nil {
		return nil, err
	}
	if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
		return nil, err
	}
	var obj *raw.Object
	var err error
	setClientHeader(call.Header())
	err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err })
	if err != nil {
		return nil, err
	}
	return newObject(obj), nil
}
<commit_msg>Fix unreachable code<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"errors"
	"fmt"

	"golang.org/x/net/context"
	raw "google.golang.org/api/storage/v1"
)

// CopierFrom creates a Copier that can copy src to dst.
// You can immediately call Run on the returned Copier, or
// you can configure it first.
func (dst *ObjectHandle) CopierFrom(src *ObjectHandle) *Copier {
	return &Copier{dst: dst, src: src}
}

// A Copier copies a source object to a destination.
type Copier struct {
	// ObjectAttrs are optional attributes to set on the destination object.
	// Any attributes must be initialized before any calls on the Copier. Nil
	// or zero-valued attributes are ignored.
	ObjectAttrs

	// RewriteToken can be set before calling Run to resume a copy
	// operation. After Run returns a non-nil error, RewriteToken will
	// have been updated to contain the value needed to resume the copy.
	RewriteToken string

	// ProgressFunc can be used to monitor the progress of a multi-RPC copy
	// operation. If ProgressFunc is not nil and CopyFrom requires multiple
	// calls to the underlying service (see
	// https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite), then
	// ProgressFunc will be invoked after each call with the number of bytes of
	// content copied so far and the total size in bytes of the source object.
	//
	// ProgressFunc is intended to make upload progress available to the
	// application. For example, the implementation of ProgressFunc may update
	// a progress bar in the application's UI, or log the result of
	// float64(copiedBytes)/float64(totalBytes).
	//
	// ProgressFunc should return quickly without blocking.
	ProgressFunc func(copiedBytes, totalBytes uint64)

	dst, src *ObjectHandle
}

// Run performs the copy.
func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) {
	if err := c.src.validate(); err != nil {
		return nil, err
	}
	if err := c.dst.validate(); err != nil {
		return nil, err
	}
	// Convert destination attributes to raw form, omitting the bucket.
	// If the bucket is included but name or content-type aren't, the service
	// returns a 400 with "Required" as the only message. Omitting the bucket
	// does not cause any problems.
	rawObject := c.ObjectAttrs.toRawObject("")
	for {
		res, err := c.callRewrite(ctx, c.src, rawObject)
		if err != nil {
			return nil, err
		}
		if c.ProgressFunc != nil {
			c.ProgressFunc(res.TotalBytesRewritten, res.ObjectSize)
		}
		if res.Done { // Finished successfully.
			return newObject(res.Resource), nil
		}
	}
}

func (c *Copier) callRewrite(ctx context.Context, src *ObjectHandle, rawObj *raw.Object) (*raw.RewriteResponse, error) {
	call := c.dst.c.raw.Objects.Rewrite(src.bucket, src.object, c.dst.bucket, c.dst.object, rawObj)

	call.Context(ctx).Projection("full")
	if c.RewriteToken != "" {
		call.RewriteToken(c.RewriteToken)
	}
	if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil {
		return nil, err
	}
	if err := applySourceConds(c.src.gen, c.src.conds, call); err != nil {
		return nil, err
	}
	if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
		return nil, err
	}
	if err := setEncryptionHeaders(call.Header(), c.src.encryptionKey, true); err != nil {
		return nil, err
	}
	var res *raw.RewriteResponse
	var err error
	setClientHeader(call.Header())
	err = runWithRetry(ctx, func() error { res, err = call.Do(); return err })
	if err != nil {
		return nil, err
	}
	c.RewriteToken = res.RewriteToken
	return res, nil
}

// ComposerFrom creates a Composer that can compose srcs into dst.
// You can immediately call Run on the returned Composer, or you can
// configure it first.
//
// The encryption key for the destination object will be used to decrypt all
// source objects and encrypt the destination object. It is an error
// to specify an encryption key for any of the source objects.
func (dst *ObjectHandle) ComposerFrom(srcs ...*ObjectHandle) *Composer {
	return &Composer{dst: dst, srcs: srcs}
}

// A Composer composes source objects into a destination object.
type Composer struct {
	// ObjectAttrs are optional attributes to set on the destination object.
	// Any attributes must be initialized before any calls on the Composer. Nil
	// or zero-valued attributes are ignored.
	ObjectAttrs

	dst  *ObjectHandle
	srcs []*ObjectHandle
}

// Run performs the compose operation.
func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) {
	if err := c.dst.validate(); err != nil {
		return nil, err
	}
	if len(c.srcs) == 0 {
		return nil, errors.New("storage: at least one source object must be specified")
	}

	req := &raw.ComposeRequest{}
	// Compose requires a non-empty Destination, so we always set it,
	// even if the caller-provided ObjectAttrs is the zero value.
	req.Destination = c.ObjectAttrs.toRawObject(c.dst.bucket)
	for _, src := range c.srcs {
		if err := src.validate(); err != nil {
			return nil, err
		}
		if src.bucket != c.dst.bucket {
			return nil, fmt.Errorf("storage: all source objects must be in bucket %q, found %q", c.dst.bucket, src.bucket)
		}
		if src.encryptionKey != nil {
			return nil, fmt.Errorf("storage: compose source %s.%s must not have encryption key", src.bucket, src.object)
		}
		srcObj := &raw.ComposeRequestSourceObjects{
			Name: src.object,
		}
		if err := applyConds("ComposeFrom source", src.gen, src.conds, composeSourceObj{srcObj}); err != nil {
			return nil, err
		}
		req.SourceObjects = append(req.SourceObjects, srcObj)
	}

	call := c.dst.c.raw.Objects.Compose(c.dst.bucket, c.dst.object, req).Context(ctx)
	if err := applyConds("ComposeFrom destination", c.dst.gen, c.dst.conds, call); err != nil {
		return nil, err
	}
	if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
		return nil, err
	}
	var obj *raw.Object
	var err error
	setClientHeader(call.Header())
	err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err })
	if err != nil {
		return nil, err
	}
	return newObject(obj), nil
}
<|endoftext|>


<commit_before>package app

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/google/gnxi/utils/xpath"
	"github.com/manifoldco/promptui"
	"github.com/openconfig/goyang/pkg/yang"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

type pathGenOpts struct {
	search     bool
	withDescr  bool
	withTypes  bool
	withPrefix bool
	pathType   string
	stateOnly  bool
	configOnly bool
	json       bool
}

type generatedPath struct {
	Path        string `json:"path,omitempty"`
	Type        string `json:"type,omitempty"`
	Description string `json:"description,omitempty"`
	Default     string `json:"default,omitempty"`
	IsState     bool   `json:"is-state,omitempty"`
}

func (a *App) PathCmdRun(d, f, e []string, pgo pathGenOpts) error {
	err := a.GenerateYangSchema(d, f, e)
	if err != nil {
		return err
	}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	out := make(chan *generatedPath)
	gpaths := make([]*generatedPath, 0)

	go func(ctx context.Context, out chan *generatedPath) {
		for {
			select {
			case m, ok := <-out:
				if !ok {
					return
				}
				gpaths = append(gpaths, m)
			case <-ctx.Done():
				return
			}
		}
	}(ctx, out)

	collected := make([]*yang.Entry, 0, 256)
	for _, entry := range a.SchemaTree.Dir {
		collected = append(collected, collectSchemaNodes(entry, true)...)
	}
	for _, entry := range collected {
		if !pgo.stateOnly && !pgo.configOnly || pgo.stateOnly && pgo.configOnly {
			out <- a.generatePath(entry, pgo.withPrefix, pgo.pathType)
			continue
		}
		state := isState(entry)
		if state && pgo.stateOnly {
			out <- a.generatePath(entry, pgo.withPrefix, pgo.pathType)
			continue
		}
		if !state && pgo.configOnly {
			out <- a.generatePath(entry, pgo.withPrefix, pgo.pathType)
			continue
		}
	}
	close(out)
	sort.Slice(gpaths, func(i, j int) bool {
		return gpaths[i].Path < gpaths[j].Path
	})
	if pgo.json {
		b, err := json.MarshalIndent(gpaths, "", "  ")
		if err != nil {
			return err
		}
		fmt.Fprintln(os.Stdout, string(b))
		return nil
	}

	if len(gpaths) == 0 {
		return errors.New("no results found")
	}

	// regular print
	if !pgo.search {
		sb := new(strings.Builder)
		for _, gp := range gpaths {
			sb.Reset()
			sb.WriteString(gp.Path)
			if pgo.withTypes {
				sb.WriteString("\t(type=")
				sb.WriteString(gp.Type)
				sb.WriteString(")")
			}
			if pgo.withDescr {
				sb.WriteString("\n")
				sb.WriteString(indent("\t", gp.Description))
			}
			fmt.Fprintln(os.Stdout, sb.String())
		}
		return nil
	}
	// search
	paths := make([]string, 0, len(gpaths))
	for _, gp := range gpaths {
		paths = append(paths, gp.Path)
	}
	p := promptui.Select{
		Label:        "select path",
		Items:        paths,
		Size:         10,
		Stdout:       os.Stdout,
		HideSelected: true,
		Searcher: func(input string, index int) bool {
			kws := strings.Split(input, " ")
			result := true
			for _, kw := range kws {
				if strings.HasPrefix(kw, "!") {
					kw = strings.TrimLeft(kw, "!")
					if kw == "" {
						continue
					}
					result = result && !strings.Contains(paths[index], kw)
				} else {
					result = result && strings.Contains(paths[index], kw)
				}
			}
			return result
		},
		Keys: &promptui.SelectKeys{
			Prev:     promptui.Key{Code: promptui.KeyPrev, Display: promptui.KeyPrevDisplay},
			Next:     promptui.Key{Code: promptui.KeyNext, Display: promptui.KeyNextDisplay},
			PageUp:   promptui.Key{Code: promptui.KeyBackward, Display: promptui.KeyBackwardDisplay},
			PageDown: promptui.Key{Code: promptui.KeyForward, Display: promptui.KeyForwardDisplay},
			Search:   promptui.Key{Code: ':', Display: ":"},
		},
	}
	index, selected, err := p.Run()
	if err != nil {
		return err
	}
	fmt.Println(selected)
	fmt.Println(a.generateTypeInfo(collected[index]))

	return nil
}

func (a *App) PathPreRunE(cmd *cobra.Command, args []string) error {
	a.Config.SetLocalFlagsFromFile(cmd)
	if a.Config.PathSearch && a.Config.PathWithDescr {
		return errors.New("flags --search and --descr cannot be used together")
	}
	if a.Config.LocalFlags.PathPathType != "xpath" && a.Config.LocalFlags.PathPathType != "gnmi" {
		return errors.New("path-type must be one of 'xpath' or 'gnmi'")
	}
	return a.yangFilesPreProcessing()
}

func (a *App) PathRunE(cmd *cobra.Command, args []string) error {
	return a.PathCmdRun(
		a.Config.GlobalFlags.Dir,
		a.Config.GlobalFlags.File,
		a.Config.GlobalFlags.Exclude,
		pathGenOpts{
			search:     a.Config.LocalFlags.PathSearch,
			withDescr:  a.Config.LocalFlags.PathWithDescr,
			withTypes:  a.Config.LocalFlags.PathWithTypes,
			withPrefix: a.Config.LocalFlags.PathWithPrefix,
			pathType:   a.Config.LocalFlags.PathPathType,
			stateOnly:  a.Config.LocalFlags.PathState,
			configOnly: a.Config.LocalFlags.PathConfig,
		},
	)
}

func (a *App) InitPathFlags(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&a.Config.LocalFlags.PathPathType, "path-type", "", "xpath", "path type xpath or gnmi")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathWithDescr, "descr", "", false, "print leaf description")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathWithPrefix, "with-prefix", "", false, "include module/submodule prefix in path elements")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathWithTypes, "types", "", false, "print leaf type")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathSearch, "search", "", false, "search through path list")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathState, "state-only", "", false, "generate paths only for YANG leafs representing state data")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathConfig, "config-only", "", false, "generate paths only for YANG leafs representing config data")
	cmd.LocalFlags().VisitAll(func(flag *pflag.Flag) {
		a.Config.FileConfig.BindPFlag(fmt.Sprintf("%s-%s", cmd.Name(), flag.Name), flag)
	})
}

func collectSchemaNodes(e *yang.Entry, leafOnly bool) []*yang.Entry {
	if e == nil {
		return []*yang.Entry{}
	}
	collected := make([]*yang.Entry, 0, 128)
	for _, child := range e.Dir {
		collected = append(collected,
			collectSchemaNodes(child, leafOnly)...)
	}
	if e.Parent != nil {
		switch {
		case e.Dir == nil && e.ListAttr != nil: // leaf-list
			fallthrough
		case e.Dir == nil: // leaf
			collected = append(collected, e)
		case e.ListAttr != nil: // list
			fallthrough
		default: // container
			if !leafOnly {
				collected = append(collected, e)
			}
		}
	}
	return collected
}

func (a *App) generatePath(entry *yang.Entry, prefixTagging bool, pType string) *generatedPath {
	gp := new(generatedPath)
	for e := entry; e != nil && e.Parent != nil; e = e.Parent {
		if e.IsCase() || e.IsChoice() {
			continue
		}
		elementName := e.Name
		if prefixTagging && e.Prefix != nil {
			elementName = fmt.Sprintf("%s:%s", e.Prefix.Name, elementName)
		}
		if e.Key != "" {
			for _, k := range strings.Fields(e.Key) {
				if prefixTagging && e.Prefix != nil {
					k = fmt.Sprintf("%s:%s", e.Prefix.Name, k)
				}
				elementName = fmt.Sprintf("%s[%s=*]", elementName, k)
			}
		}
		gp.Path = fmt.Sprintf("/%s%s", elementName, gp.Path)
	}

	gp.Description = entry.Description
	gp.Type = entry.Type.Name
	gp.Default = entry.DefaultValue()
	gp.IsState = isState(entry)
	if pType == "gnmi" {
		gnmiPath, err := xpath.ToGNMIPath(gp.Path)
		if err != nil {
			fmt.Fprintf(os.Stderr, "path: %s could not be changed to gnmi format: %v\n", gp.Path, err)
		}
		gp.Path = gnmiPath.String()
	}
	return gp
}

func (a *App) generateTypeInfo(e *yang.Entry) string {
	if e == nil || e.Type == nil {
		return "unknown type"
	}
	t := e.Type
	rstr := fmt.Sprintf("- type: %s", t.Kind)
	switch t.Kind {
	case yang.Ybits:
		data := getAnnotation(e, "bits")
		if data != nil {
			rstr += fmt.Sprintf(" %v", data)
		}
	case yang.Yenum:
		data := getAnnotation(e, "enum")
		if data != nil {
			rstr += fmt.Sprintf(" %v", data)
		}
	case yang.Yleafref:
		rstr += fmt.Sprintf(" %q", t.Path)
	case yang.Yidentityref:
		rstr += fmt.Sprintf(" %q", t.IdentityBase.Name)
		if a.Config.LocalFlags.PathWithPrefix {
			data := getAnnotation(e, "prefix-qualified-identities")
			if data != nil {
				rstr += fmt.Sprintf(" %v", data)
			}
		} else {
			identities := make([]string, 0, 64)
			for i := range t.IdentityBase.Values {
				identities = append(identities, t.IdentityBase.Values[i].Name)
			}
			rstr += fmt.Sprintf(" %v", identities)
		}

	case yang.Yunion:
		unionlist := make([]string, 0, len(t.Type))
		for i := range t.Type {
			unionlist = append(unionlist, t.Type[i].Name)
		}
		rstr += fmt.Sprintf(" %v", unionlist)
	default:
	}
	rstr += "\n"

	if t.Root != nil {
		data := getAnnotation(e, "root.type")
		if data != nil && t.Kind.String() != data.(string) {
			rstr += fmt.Sprintf("- root.type: %v\n", data)
		}
	}
	if t.Units != "" {
		rstr += fmt.Sprintf("- units: %s\n", t.Units)
	}
	if t.Default != "" {
		rstr += fmt.Sprintf("- default: %q\n", t.Default)
	}
	if t.FractionDigits != 0 {
		rstr += fmt.Sprintf("- fraction-digits: %d\n", t.FractionDigits)
	}
	if len(t.Length) > 0 {
		rstr += fmt.Sprintf("- length: %s\n", t.Length)
	}
	if t.Kind == yang.YinstanceIdentifier && !t.OptionalInstance {
		rstr += "- required\n"
	}

	if len(t.Pattern) > 0 {
		rstr += fmt.Sprintf("- pattern: %s\n", strings.Join(t.Pattern, "|"))
	}
	b := yang.BaseTypedefs[t.Kind.String()].YangType
	if len(t.Range) > 0 && !t.Range.Equal(b.Range) {
		rstr += fmt.Sprintf("- range: %s\n", t.Range)
	}
	return rstr
}

func getAnnotation(entry *yang.Entry, name string) interface{} {
	if entry.Annotation != nil {
		data, ok := entry.Annotation[name]
		if ok {
			return data
		}
	}
	return nil
}

func isState(e *yang.Entry) bool {
	if e.Config == yang.TSFalse {
		return true
	}
	if e.Parent != nil {
		return isState(e.Parent)
	}
	return false
}
<commit_msg>add namespace to json output<commit_after>package app

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/google/gnxi/utils/xpath"
	"github.com/manifoldco/promptui"
	"github.com/openconfig/goyang/pkg/yang"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

type pathGenOpts struct {
	search     bool
	withDescr  bool
	withTypes  bool
	withPrefix bool
	pathType   string
	stateOnly  bool
	configOnly bool
	json       bool
}

type generatedPath struct {
	Path        string `json:"path,omitempty"`
	Type        string `json:"type,omitempty"`
	Description string `json:"description,omitempty"`
	Default     string `json:"default,omitempty"`
	IsState     bool   `json:"is-state,omitempty"`
	Namespace   string `json:"namespace,omitempty"`
}

func (a *App) PathCmdRun(d, f, e []string, pgo pathGenOpts) error {
	err := a.GenerateYangSchema(d, f, e)
	if err != nil {
		return err
	}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	out := make(chan *generatedPath)
	gpaths := make([]*generatedPath, 0)

	go func(ctx context.Context, out chan *generatedPath) {
		for {
			select {
			case m, ok := <-out:
				if !ok {
					return
				}
				gpaths = append(gpaths, m)
			case <-ctx.Done():
				return
			}
		}
	}(ctx, out)

	collected := make([]*yang.Entry, 0, 256)
	for _, entry := range a.SchemaTree.Dir {
		collected = append(collected, collectSchemaNodes(entry, true)...)
	}
	for _, entry := range collected {
		if !pgo.stateOnly && !pgo.configOnly || pgo.stateOnly && pgo.configOnly {
			out <- a.generatePath(entry, pgo.withPrefix, pgo.pathType)
			continue
		}
		state := isState(entry)
		if state && pgo.stateOnly {
			out <- a.generatePath(entry, pgo.withPrefix, pgo.pathType)
			continue
		}
		if !state && pgo.configOnly {
			out <- a.generatePath(entry, pgo.withPrefix, pgo.pathType)
			continue
		}
	}
	close(out)
	sort.Slice(gpaths, func(i, j int) bool {
		return gpaths[i].Path < gpaths[j].Path
	})
	if pgo.json {
		b, err := json.MarshalIndent(gpaths, "", "  ")
		if err != nil {
			return err
		}
		fmt.Fprintln(os.Stdout, string(b))
		return nil
	}

	if len(gpaths) == 0 {
		return errors.New("no results found")
	}

	// regular print
	if !pgo.search {
		sb := new(strings.Builder)
		for _, gp := range gpaths {
			sb.Reset()
			sb.WriteString(gp.Path)
			if pgo.withTypes {
				sb.WriteString("\t(type=")
				sb.WriteString(gp.Type)
				sb.WriteString(")")
			}
			if pgo.withDescr {
				sb.WriteString("\n")
				sb.WriteString(indent("\t", gp.Description))
			}
			fmt.Fprintln(os.Stdout, sb.String())
		}
		return nil
	}
	// search
	paths := make([]string, 0, len(gpaths))
	for _, gp := range gpaths {
		paths = append(paths, gp.Path)
	}
	p := promptui.Select{
		Label:        "select path",
		Items:        paths,
		Size:         10,
		Stdout:       os.Stdout,
		HideSelected: true,
		Searcher: func(input string, index int) bool {
			kws := strings.Split(input, " ")
			result := true
			for _, kw := range kws {
				if strings.HasPrefix(kw, "!") {
					kw = strings.TrimLeft(kw, "!")
					if kw == "" {
						continue
					}
					result = result && !strings.Contains(paths[index], kw)
				} else {
					result = result && strings.Contains(paths[index], kw)
				}
			}
			return result
		},
		Keys: &promptui.SelectKeys{
			Prev:     promptui.Key{Code: promptui.KeyPrev, Display: promptui.KeyPrevDisplay},
			Next:     promptui.Key{Code: promptui.KeyNext, Display: promptui.KeyNextDisplay},
			PageUp:   promptui.Key{Code: promptui.KeyBackward, Display: promptui.KeyBackwardDisplay},
			PageDown: promptui.Key{Code: promptui.KeyForward, Display: promptui.KeyForwardDisplay},
			Search:   promptui.Key{Code: ':', Display: ":"},
		},
	}
	index, selected, err := p.Run()
	if err != nil {
		return err
	}
	fmt.Println(selected)
	fmt.Println(a.generateTypeInfo(collected[index]))

	return nil
}

func (a *App) PathPreRunE(cmd *cobra.Command, args []string) error {
	a.Config.SetLocalFlagsFromFile(cmd)
	if a.Config.PathSearch && a.Config.PathWithDescr {
		return errors.New("flags --search and --descr cannot be used together")
	}
	if a.Config.LocalFlags.PathPathType != "xpath" && a.Config.LocalFlags.PathPathType != "gnmi" {
		return errors.New("path-type must be one of 'xpath' or 'gnmi'")
	}
	return a.yangFilesPreProcessing()
}

func (a *App) PathRunE(cmd *cobra.Command, args []string) error {
	return a.PathCmdRun(
		a.Config.GlobalFlags.Dir,
		a.Config.GlobalFlags.File,
		a.Config.GlobalFlags.Exclude,
		pathGenOpts{
			search:     a.Config.LocalFlags.PathSearch,
			withDescr:  a.Config.LocalFlags.PathWithDescr,
			withTypes:  a.Config.LocalFlags.PathWithTypes,
			withPrefix: a.Config.LocalFlags.PathWithPrefix,
			pathType:   a.Config.LocalFlags.PathPathType,
			stateOnly:  a.Config.LocalFlags.PathState,
			configOnly: a.Config.LocalFlags.PathConfig,
		},
	)
}

func (a *App) InitPathFlags(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&a.Config.LocalFlags.PathPathType, "path-type", "", "xpath", "path type xpath or gnmi")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathWithDescr, "descr", "", false, "print leaf description")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathWithPrefix, "with-prefix", "", false, "include module/submodule prefix in path elements")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathWithTypes, "types", "", false, "print leaf type")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathSearch, "search", "", false, "search through path list")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathState, "state-only", "", false, "generate paths only for YANG leafs representing state data")
	cmd.Flags().BoolVarP(&a.Config.LocalFlags.PathConfig, "config-only", "", false, "generate paths only for YANG leafs representing config data")
	cmd.LocalFlags().VisitAll(func(flag *pflag.Flag) {
		a.Config.FileConfig.BindPFlag(fmt.Sprintf("%s-%s", cmd.Name(), flag.Name), flag)
	})
}

func collectSchemaNodes(e *yang.Entry, leafOnly bool) []*yang.Entry {
	if e == nil {
		return []*yang.Entry{}
	}
	collected := make([]*yang.Entry, 0, 128)
	for _, child := range e.Dir {
		collected = append(collected,
			collectSchemaNodes(child, leafOnly)...)
	}
	if e.Parent != nil {
		switch {
		case e.Dir == nil && e.ListAttr != nil: // leaf-list
			fallthrough
		case e.Dir == nil: // leaf
			collected = append(collected, e)
		case e.ListAttr != nil: // list
			fallthrough
		default: // container
			if !leafOnly {
				collected = append(collected, e)
			}
		}
	}
	return collected
}

func (a *App) generatePath(entry *yang.Entry, prefixTagging bool, pType string) *generatedPath {
	gp := new(generatedPath)
	for e := entry; e != nil && e.Parent != nil; e = e.Parent {
		if e.IsCase() || e.IsChoice() {
			continue
		}
		elementName := e.Name
		if prefixTagging && e.Prefix != nil {
			elementName = fmt.Sprintf("%s:%s", e.Prefix.Name, elementName)
		}
		if e.Key != "" {
			for _, k := range strings.Fields(e.Key) {
				if prefixTagging && e.Prefix != nil {
					k = fmt.Sprintf("%s:%s", e.Prefix.Name, k)
				}
				elementName = fmt.Sprintf("%s[%s=*]", elementName, k)
			}
		}
		gp.Path = fmt.Sprintf("/%s%s", elementName, gp.Path)
	}

	gp.Description = entry.Description
	gp.Type = entry.Type.Name
	gp.Default = entry.DefaultValue()
	gp.IsState = isState(entry)
	gp.Namespace = entry.Namespace().NName()
	if pType == "gnmi" {
		gnmiPath, err := xpath.ToGNMIPath(gp.Path)
		if err != nil {
			fmt.Fprintf(os.Stderr, "path: %s could not be changed to gnmi format: %v\n", gp.Path, err)
		}
		gp.Path = gnmiPath.String()
	}
	return gp
}

func (a *App) generateTypeInfo(e *yang.Entry) string {
	if e == nil || e.Type == nil {
		return "unknown type"
	}
	t := e.Type
	rstr := fmt.Sprintf("- type: %s", t.Kind)
	switch t.Kind {
	case yang.Ybits:
		data := getAnnotation(e, "bits")
		if data != nil {
			rstr += fmt.Sprintf(" %v", data)
		}
	case yang.Yenum:
		data := getAnnotation(e, "enum")
		if data != nil {
			rstr += fmt.Sprintf(" %v", data)
		}
	case yang.Yleafref:
		rstr += fmt.Sprintf(" %q", t.Path)
	case yang.Yidentityref:
		rstr += fmt.Sprintf(" %q", t.IdentityBase.Name)
		if a.Config.LocalFlags.PathWithPrefix {
			data := getAnnotation(e, "prefix-qualified-identities")
			if data != nil {
				rstr += fmt.Sprintf(" %v", data)
			}
		} else {
			identities := make([]string, 0, 64)
			for i := range t.IdentityBase.Values {
				identities = append(identities, t.IdentityBase.Values[i].Name)
			}
			rstr += fmt.Sprintf(" %v", identities)
		}

	case yang.Yunion:
		unionlist := make([]string, 0, len(t.Type))
		for i := range t.Type {
			unionlist = append(unionlist, t.Type[i].Name)
		}
		rstr += fmt.Sprintf(" %v", unionlist)
	default:
	}
	rstr += "\n"

	if t.Root != nil {
		data := getAnnotation(e, "root.type")
		if data != nil && t.Kind.String() != data.(string) {
			rstr += fmt.Sprintf("- root.type: %v\n", data)
		}
	}
	if t.Units != "" {
		rstr += fmt.Sprintf("- units: %s\n", t.Units)
	}
	if t.Default != "" {
		rstr += fmt.Sprintf("- default: %q\n", t.Default)
	}
	if t.FractionDigits != 0 {
		rstr += fmt.Sprintf("- fraction-digits: %d\n", t.FractionDigits)
	}
	if len(t.Length) > 0 {
		rstr += fmt.Sprintf("- length: %s\n", t.Length)
	}
	if t.Kind == yang.YinstanceIdentifier && !t.OptionalInstance {
		rstr += "- required\n"
	}

	if len(t.Pattern) > 0 {
		rstr += fmt.Sprintf("- pattern: %s\n", strings.Join(t.Pattern, "|"))
	}
	b := yang.BaseTypedefs[t.Kind.String()].YangType
	if len(t.Range) > 0 && !t.Range.Equal(b.Range) {
		rstr += fmt.Sprintf("- range: %s\n", t.Range)
	}
	return rstr
}

func getAnnotation(entry *yang.Entry, name string) interface{} {
	if entry.Annotation != nil {
		data, ok := entry.Annotation[name]
		if ok {
			return data
		}
	}
	return nil
}

func isState(e *yang.Entry) bool {
	if e.Config == yang.TSFalse {
		return true
	}
	if e.Parent != nil {
		return isState(e.Parent)
	}
	return false
}
<|endoftext|>