<commit_before>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"path"
	"strconv"
	"strings"
	"time"
	"unicode"
)

// PageType is a convenience type alias for page type
type PageType string

const (
	// Post - A regular blog article
	Post PageType = "Post"
	// Page - A site-wide page
	Page = "Page"
	// Snippet - Twitter-like short blog post
	Snippet = "Snippet"
)

const (
	// DefaultDateFormat is the default format used in posts and templates
	DefaultDateFormat string = time.RFC3339
	// IFTTTDateFormat is a date format IFTTT uses for recipe ingredients in actions
	IFTTTDateFormat = "January 02, 2006 at 03:04PM"
)

// Article represents a blogger post, page or a snippet. Contains information useful for blog publishing.
type Article struct {
	Author       string
	DateModified *time.Time
	DateUpdated  *time.Time
	Title        string
	Content      string
	RawContent   []byte
	Description  string
	Filename     string
	Link         string
	Identifier   string
	Snippet      bool
	Type         PageType
	Draft        bool
	Tags         []string
	AppID        string
	Meta         map[string]string
}

// HasTag checks if the given article contains a certain tag
func (a Article) HasTag(aTag string) bool {
	for _, tag := range a.Tags {
		if aTag == tag {
			return true
		}
	}

	return false
}

// BasePath returns a base path for the given article, relative to blog root path
func (a Article) BasePath() string {
	switch a.Type {
	case Post, Snippet:
		if a.Draft {
			return "drafts"
		}

	case Page:
		return ""
	}

	return path.Join(strconv.Itoa(a.DateModified.Year()), fmt.Sprintf("%02d", int(a.DateModified.Month())))
}

// FullPath combines BasePath with articles file name
func (a Article) FullPath() string {
	return path.Join(a.BasePath(), a.Filename)
}

// Articles is an convenience type alias for article slice
type Articles []*Article

func (a Articles) Len() int {
	return len(a)
}

func (a Articles) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a Articles) Less(i, j int) bool {
	left := *(a[i].DateModified)
	right := *(a[j].DateModified)

	return right.Before(left)
}

// Print sends an article header in plain text to standard output
func (a Article) Print() {
	fmt.Println("---")

	var articleType string

	switch a.Type {
	case Post:
		articleType = "Post"
	case Snippet:
		articleType = "Snippet"
	case Page:
		articleType = "Page"
	}

	if a.Type != Snippet {
		fmt.Printf("title: %s\n", a.Title)
	}

	fmt.Printf("author: %s\n", a.Author)
	fmt.Printf("type: %s\n", articleType)
	fmt.Println("tags: ")
	if a.DateModified != nil {
		fmt.Printf("date: %v\n", a.DateModified.Format(DefaultDateFormat))
	}

	if a.DateUpdated != nil {
		fmt.Printf("date: %v\n", a.DateUpdated.Format(DefaultDateFormat))
	}

	if len(a.AppID) > 0 {
		fmt.Printf("appid: %v\n", a.AppID)
	}

	if a.Draft {
		fmt.Printf("draft: true")
	}

	if a.Meta != nil {
	}

	fmt.Println("---")
	fmt.Println("")
}

// ParseFrontMatter reads the front matter-type article header
func ParseFrontMatter(reader *bufio.Reader) (map[string]string, error) {

	data := make(map[string]string)

	line, lineErr := reader.ReadString('\n')

	if !strings.HasPrefix(line, "---") {
		return data, errors.New("Invalid front matter header")
	}

	for {
		line, lineErr = reader.ReadString('\n')
		if lineErr != nil {
			break
		}

		if strings.HasPrefix(line, "---") {
			break
		}

		values := strings.SplitN(line, ":", 2)

		if len(values) < 2 {
			continue
		}

		key, value := values[0], values[1]

		key = strings.Trim(key, " \t\r\n")
		value = strings.Trim(value, " \t\r\n")

		data[key] = value
	}

	return data, nil
}

// ReadArticle returns an article read from a Reader
func ReadArticle(reader *bufio.Reader) (Article, error) {
	article := Article{}

	frontMatter, matterErr := ParseFrontMatter(reader)

	if matterErr != nil {
		return article, errors.New("Invalid article header")
	}

	for key, value := range frontMatter {

		if strings.HasPrefix(key, "meta-") {
			if article.Meta == nil {
				article.Meta = make(map[string]string)
			}
			metaName := strings.TrimPrefix(key, "meta-")
			article.Meta[metaName] = value
			continue
		}

		switch key {
		case "title":
			article.Title = value
		case "author":
			article.Author = value
		case "description":
			article.Description = value
		case "link":
			article.Link = value
		case "date":
			dateStr := value
			modTime, timeErr := time.Parse(DefaultDateFormat, dateStr)
			if timeErr == nil {
				article.DateModified = &modTime
				break
			}

			modTime, timeErr = time.Parse(IFTTTDateFormat, dateStr)
			if timeErr == nil {
				article.DateModified = &modTime
				break
			}

			if timeErr != nil {
				return article, timeErr
			}

		case "updated":
			dateStr := value
			modTime, timeErr := time.Parse(DefaultDateFormat, dateStr)
			if timeErr == nil {
				article.DateUpdated = &modTime
				break
			}

			modTime, timeErr = time.Parse(IFTTTDateFormat, dateStr)
			if timeErr == nil {
				article.DateUpdated = &modTime
				break
			}

			if timeErr != nil {
				return article, timeErr
			}

		case "appid":
			article.AppID = value
		case "draft":
			article.Draft = (value == "true")
		case "type":
			switch value {
			case "Post":
				article.Type = Post
			case "Page":
				article.Type = Page
			case "Snippet":
				article.Type = Snippet
			}

		case "tags":
			fieldsFunc := func(divider rune) bool {
				return unicode.IsSpace(divider) || divider == ',' || divider == ';'
			}

			for _, tag := range strings.FieldsFunc(value, fieldsFunc) {
				article.Tags = append(article.Tags, strings.ToLower(tag))
			}
		}
	}

	if article.DateModified == nil {
		now := time.Now()
		article.DateModified = &now
	}

	var contentBuffer bytes.Buffer
	reader.WriteTo(&contentBuffer)

	article.RawContent = contentBuffer.Bytes()

	return article, nil
}
<commit_msg>Added a missing newline in printed-out post template.<commit_after>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"path"
	"strconv"
	"strings"
	"time"
	"unicode"
)

// PageType is a convenience type alias for page type
type PageType string

const (
	// Post - A regular blog article
	Post PageType = "Post"
	// Page - A site-wide page
	Page = "Page"
	// Snippet - Twitter-like short blog post
	Snippet = "Snippet"
)

const (
	// DefaultDateFormat is the default format used in posts and templates
	DefaultDateFormat string = time.RFC3339
	// IFTTTDateFormat is a date format IFTTT uses for recipe ingredients in actions
	IFTTTDateFormat = "January 02, 2006 at 03:04PM"
)

// Article represents a blogger post, page or a snippet. Contains information useful for blog publishing.
type Article struct {
	Author       string
	DateModified *time.Time
	DateUpdated  *time.Time
	Title        string
	Content      string
	RawContent   []byte
	Description  string
	Filename     string
	Link         string
	Identifier   string
	Snippet      bool
	Type         PageType
	Draft        bool
	Tags         []string
	AppID        string
	Meta         map[string]string
}

// HasTag checks if the given article contains a certain tag
func (a Article) HasTag(aTag string) bool {
	for _, tag := range a.Tags {
		if aTag == tag {
			return true
		}
	}

	return false
}

// BasePath returns a base path for the given article, relative to blog root path
func (a Article) BasePath() string {
	switch a.Type {
	case Post, Snippet:
		if a.Draft {
			return "drafts"
		}

	case Page:
		return ""
	}

	return path.Join(strconv.Itoa(a.DateModified.Year()), fmt.Sprintf("%02d", int(a.DateModified.Month())))
}

// FullPath combines BasePath with articles file name
func (a Article) FullPath() string {
	return path.Join(a.BasePath(), a.Filename)
}

// Articles is an convenience type alias for article slice
type Articles []*Article

func (a Articles) Len() int {
	return len(a)
}

func (a Articles) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a Articles) Less(i, j int) bool {
	left := *(a[i].DateModified)
	right := *(a[j].DateModified)

	return right.Before(left)
}

// Print sends an article header in plain text to standard output
func (a Article) Print() {
	fmt.Println("---")

	var articleType string

	switch a.Type {
	case Post:
		articleType = "Post"
	case Snippet:
		articleType = "Snippet"
	case Page:
		articleType = "Page"
	}

	if a.Type != Snippet {
		fmt.Printf("title: %s\n", a.Title)
	}

	fmt.Printf("author: %s\n", a.Author)
	fmt.Printf("type: %s\n", articleType)
	fmt.Println("tags: ")
	if a.DateModified != nil {
		fmt.Printf("date: %v\n", a.DateModified.Format(DefaultDateFormat))
	}

	if a.DateUpdated != nil {
		fmt.Printf("date: %v\n", a.DateUpdated.Format(DefaultDateFormat))
	}

	if len(a.AppID) > 0 {
		fmt.Printf("appid: %v\n", a.AppID)
	}

	if a.Draft {
		fmt.Printf("draft: true\n")
	}

	if a.Meta != nil {
	}

	fmt.Println("---")
	fmt.Println("")
}

// ParseFrontMatter reads the front matter-type article header
func ParseFrontMatter(reader *bufio.Reader) (map[string]string, error) {

	data := make(map[string]string)

	line, lineErr := reader.ReadString('\n')

	if !strings.HasPrefix(line, "---") {
		return data, errors.New("Invalid front matter header")
	}

	for {
		line, lineErr = reader.ReadString('\n')
		if lineErr != nil {
			break
		}

		if strings.HasPrefix(line, "---") {
			break
		}

		values := strings.SplitN(line, ":", 2)

		if len(values) < 2 {
			continue
		}

		key, value := values[0], values[1]

		key = strings.Trim(key, " \t\r\n")
		value = strings.Trim(value, " \t\r\n")

		data[key] = value
	}

	return data, nil
}

// ReadArticle returns an article read from a Reader
func ReadArticle(reader *bufio.Reader) (Article, error) {
	article := Article{}

	frontMatter, matterErr := ParseFrontMatter(reader)

	if matterErr != nil {
		return article, errors.New("Invalid article header")
	}

	for key, value := range frontMatter {

		if strings.HasPrefix(key, "meta-") {
			if article.Meta == nil {
				article.Meta = make(map[string]string)
			}
			metaName := strings.TrimPrefix(key, "meta-")
			article.Meta[metaName] = value
			continue
		}

		switch key {
		case "title":
			article.Title = value
		case "author":
			article.Author = value
		case "description":
			article.Description = value
		case "link":
			article.Link = value
		case "date":
			dateStr := value
			modTime, timeErr := time.Parse(DefaultDateFormat, dateStr)
			if timeErr == nil {
				article.DateModified = &modTime
				break
			}

			modTime, timeErr = time.Parse(IFTTTDateFormat, dateStr)
			if timeErr == nil {
				article.DateModified = &modTime
				break
			}

			if timeErr != nil {
				return article, timeErr
			}

		case "updated":
			dateStr := value
			modTime, timeErr := time.Parse(DefaultDateFormat, dateStr)
			if timeErr == nil {
				article.DateUpdated = &modTime
				break
			}

			modTime, timeErr = time.Parse(IFTTTDateFormat, dateStr)
			if timeErr == nil {
				article.DateUpdated = &modTime
				break
			}

			if timeErr != nil {
				return article, timeErr
			}

		case "appid":
			article.AppID = value
		case "draft":
			article.Draft = (value == "true")
		case "type":
			switch value {
			case "Post":
				article.Type = Post
			case "Page":
				article.Type = Page
			case "Snippet":
				article.Type = Snippet
			}

		case "tags":
			fieldsFunc := func(divider rune) bool {
				return unicode.IsSpace(divider) || divider == ',' || divider == ';'
			}

			for _, tag := range strings.FieldsFunc(value, fieldsFunc) {
				article.Tags = append(article.Tags, strings.ToLower(tag))
			}
		}
	}

	if article.DateModified == nil {
		now := time.Now()
		article.DateModified = &now
	}

	var contentBuffer bytes.Buffer
	reader.WriteTo(&contentBuffer)

	article.RawContent = contentBuffer.Bytes()

	return article, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package assert

import "github.com/reborndb/reborn/extern/redis-port/pkg/libs/log"

func Must(b bool) {
	if b {
		return
	}
	log.Panic("assertion failed")
}

func ErrorIsNil(err error) {
	if err == nil {
		return
	}
	log.PanicError(err, "error happens, assertion failed")
}
<commit_msg>remove assert pkg, use testing/assert<commit_after><|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"
	"os"
	"path"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

type DirInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	id           fuseops.InodeID
	implicitDirs bool

	// The the GCS object backing the inode. The object's name is used as a
	// prefix when listing. Special case: the empty string means this is the root
	// inode.
	//
	// INVARIANT: src != nil
	// INVARIANT: src.Name == "" || src.Name[len(name)-1] == '/'
	src gcs.Object

	/////////////////////////
	// Mutable state
	/////////////////////////

	lc lookupCount
}

var _ Inode = &DirInode{}

// Set notes on NewRootInode.
const RootGen int64 = 0

// Create a directory inode for the root of the file system. For this inode,
// the result of SourceGeneration() is guaranteed to be RootGen.
//
// The initial lookup count is one.
func NewRootInode(
	bucket gcs.Bucket,
	implicitDirs bool) (d *DirInode) {
	dummy := &gcs.Object{
		Name:       "",
		Generation: RootGen,
	}

	d = NewDirInode(bucket, fuseops.RootInodeID, dummy, implicitDirs)
	return
}

// Create a directory inode for the supplied source object. The object's name
// must end with a slash unless this is the root directory, in which case it
// must be empty.
//
// If implicitDirs is set, LookUpChild will use ListObjects to find child
// directories that are "implicitly" defined by the existence of their own
// descendents. For example, if there is an object named "foo/bar/baz" and this
// is the directory "foo", a child directory named "bar" will be implied.
//
// The initial lookup count is one.
//
// REQUIRES: o != nil
// REQUIRES: o.Name == "" || o.Name[len(o.Name)-1] == '/'
func NewDirInode(
	bucket gcs.Bucket,
	id fuseops.InodeID,
	o *gcs.Object,
	implicitDirs bool) (d *DirInode) {
	if o.Name != "" && o.Name[len(o.Name)-1] != '/' {
		panic(fmt.Sprintf("Unexpected name: %s", o.Name))
	}

	// Set up the struct.
	d = &DirInode{
		bucket:       bucket,
		id:           id,
		implicitDirs: implicitDirs,
		src:          *o,
	}

	d.lc = lookupCount{
		count:   1,
		destroy: func() error { return nil },
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (d *DirInode) checkInvariants() {
	// INVARIANT: src.Name == "" || src.Name[len(name)-1] == '/'
	if !(d.src.Name == "" || d.src.Name[len(d.src.Name)-1] == '/') {
		panic(fmt.Sprintf("Unexpected name: %s", d.src.Name))
	}
}

func (d *DirInode) clobbered(ctx context.Context) (clobbered bool, err error) {
	// Special case: the root is never clobbered.
	if d.ID() == fuseops.RootInodeID {
		return
	}

	// Stat the backing object.
	req := &gcs.StatObjectRequest{
		Name: d.Name(),
	}

	o, err := d.bucket.StatObject(ctx, req)

	// "Not found" means clobbered.
	if _, ok := err.(*gcs.NotFoundError); ok {
		clobbered = true
		err = nil
		return
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	// We are clobbered if the generation number has changed.
	clobbered = o.Generation != d.SourceGeneration()

	return
}

func (d *DirInode) lookUpChildFile(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name)
	if err != nil {
		err = fmt.Errorf("statObjectMayNotExist: %v", err)
		return
	}

	return
}

func (d *DirInode) lookUpChildDir(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Stat the placeholder object.
	b.Add(func(ctx context.Context) (err error) {
		o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name+"/")
		if err != nil {
			err = fmt.Errorf("statObjectMayNotExist: %v", err)
			return
		}

		return
	})

	// If implicit directories are enabled, find out whether the child name is
	// implicitly defined.
	var implicitlyDefined bool
	if d.implicitDirs {
		b.Add(func(ctx context.Context) (err error) {
			implicitlyDefined, err = objectNamePrefixNonEmpty(
				ctx,
				d.bucket,
				d.Name()+name+"/")

			if err != nil {
				err = fmt.Errorf("objectNamePrefixNonEmpty: %v", err)
				return
			}

			return
		})
	}

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// If statting failed by the directory is implicitly defined, fake a source
	// object.
	if o == nil && implicitlyDefined {
		o = &gcs.Object{
			Name:       d.Name() + name + "/",
			Generation: ImplicitDirGen,
		}
	}

	return
}

// List the supplied object name prefix to find out whether it is non-empty.
func objectNamePrefixNonEmpty(
	ctx context.Context,
	bucket gcs.Bucket,
	prefix string) (nonEmpty bool, err error) {
	req := &gcs.ListObjectsRequest{
		Prefix:     prefix,
		MaxResults: 1,
	}

	listing, err := bucket.ListObjects(ctx, req)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	nonEmpty = len(listing.Objects) != 0
	return
}

// Stat the object with the given name, returning (nil, nil) if the object
// doesn't exist rather than failing.
func statObjectMayNotExist(
	ctx context.Context,
	bucket gcs.Bucket,
	name string) (o *gcs.Object, err error) {
	// Call the bucket.
	req := &gcs.StatObjectRequest{
		Name: name,
	}

	o, err = bucket.StatObject(ctx, req)

	// Suppress "not found" errors.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
	}

	// Annotate others.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (d *DirInode) Lock() {
	// We don't require locks.
}

func (d *DirInode) Unlock() {
	// We don't require locks.
}

// Return the ID of this inode.
func (d *DirInode) ID() fuseops.InodeID {
	return d.id
}

// Return the full name of the directory object in GCS, including the trailing
// slash (e.g. "foo/bar/").
func (d *DirInode) Name() string {
	return d.src.Name
}

// Return the generation number from which this inode was branched.
func (d *DirInode) SourceGeneration() int64 {
	return d.src.Generation
}

// LOCKS_REQUIRED(d.mu)
func (d *DirInode) IncrementLookupCount() {
	d.lc.Inc()
}

// LOCKS_REQUIRED(d.mu)
func (d *DirInode) DecrementLookupCount(n uint64) (destroyed bool) {
	destroyed = d.lc.Dec(n)
	return
}

func (d *DirInode) Attributes(
	ctx context.Context) (attrs fuseops.InodeAttributes, err error) {
	// Find out whether the backing object has been clobbered in GCS.
	clobbered, err := d.clobbered(ctx)
	if err != nil {
		err = fmt.Errorf("clobbered: %v", err)
		return
	}

	// Set up basic attributes.
	attrs = fuseops.InodeAttributes{
		Mode: 0700 | os.ModeDir,
	}

	// Modify Nlink as appropriate.
	if !clobbered {
		attrs.Nlink = 1
	}

	return
}

// See notes on DirInode.LookUpChild.
const ImplicitDirGen int64 = -1

// A suffix that can be used to unambiguously tag a file system name.
// (Unambiguous because U+000A is not allowed in GCS object names.) This is
// used to refer to the file in a (file, directory) pair with conflicting
// object names.
//
// See also the notes on DirInode.LookUpChild.
const ConflictingFileNameSuffix = "\n"

// Look up the direct child with the given relative name, returning a record
// for the current object of that name in the GCS bucket. If both a file and a
// directory with the given name exist, the directory is preferred. Return
// fuse.ENOENT if neither is found.
//
// Special case: if the name ends in ConflictingFileNameSuffix, we strip the
// suffix, confirm that a conflicting directory exists, then return a record
// for the file.
//
// If this inode was created with implicitDirs is set, this method will use
// ListObjects to find child directories that are "implicitly" defined by the
// existence of their own descendents. For example, if there is an object named
// "foo/bar/baz" and this is the directory "foo", a child directory named "bar"
// will be implied. In this case, the child's generation will be the sentinel
// value ImplicitDirGen.
func (d *DirInode) LookUpChild(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Stat the child as a file.
	var fileRecord *gcs.Object
	b.Add(func(ctx context.Context) (err error) {
		fileRecord, err = d.lookUpChildFile(ctx, name)
		return
	})

	// Stat the child as a directory.
	var dirRecord *gcs.Object
	b.Add(func(ctx context.Context) (err error) {
		dirRecord, err = d.lookUpChildDir(ctx, name)
		return
	})

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// Prefer directories over files.
	switch {
	case dirRecord != nil:
		o = dirRecord
	case fileRecord != nil:
		o = fileRecord
	default:
		err = fuse.ENOENT
	}

	return
}

// Read some number of entries from the directory, returning a continuation
// token that can be used to pick up the read operation where it left off.
// Supply the empty token on the first call.
//
// At the end of the directory, the returned continuation token will be empty.
// Otherwise it will be non-empty. There is no guarantee about the number of
// entries returned; it may be zero even with a non-empty continuation token.
//
// The contents of the Offset and Inode fields for returned entries is
// undefined.
func (d *DirInode) ReadEntries(
	ctx context.Context,
	tok string) (entries []fuseutil.Dirent, newTok string, err error) {
	// Ask the bucket to list some objects.
	req := &gcs.ListObjectsRequest{
		Delimiter:         "/",
		Prefix:            d.Name(),
		ContinuationToken: tok,
	}

	listing, err := d.bucket.ListObjects(ctx, req)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	// Convert objects to entries for files.
	for _, o := range listing.Objects {
		e := fuseutil.Dirent{
			Name: path.Base(o.Name),
			Type: fuseutil.DT_File,
		}

		entries = append(entries, e)
	}

	// Convert runs to entries for directories.
	for _, p := range listing.CollapsedRuns {
		e := fuseutil.Dirent{
			Name: path.Base(p),
			Type: fuseutil.DT_Directory,
		}

		entries = append(entries, e)
	}

	// Return an appropriate continuation token, if any.
	newTok = listing.ContinuationToken

	return
}
<commit_msg>Added lookup support for conflict markers.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"
	"os"
	"path"
	"strings"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

type DirInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	id           fuseops.InodeID
	implicitDirs bool

	// The the GCS object backing the inode. The object's name is used as a
	// prefix when listing. Special case: the empty string means this is the root
	// inode.
	//
	// INVARIANT: src != nil
	// INVARIANT: src.Name == "" || src.Name[len(name)-1] == '/'
	src gcs.Object

	/////////////////////////
	// Mutable state
	/////////////////////////

	lc lookupCount
}

var _ Inode = &DirInode{}

// Set notes on NewRootInode.
const RootGen int64 = 0

// Create a directory inode for the root of the file system. For this inode,
// the result of SourceGeneration() is guaranteed to be RootGen.
//
// The initial lookup count is one.
func NewRootInode(
	bucket gcs.Bucket,
	implicitDirs bool) (d *DirInode) {
	dummy := &gcs.Object{
		Name:       "",
		Generation: RootGen,
	}

	d = NewDirInode(bucket, fuseops.RootInodeID, dummy, implicitDirs)
	return
}

// Create a directory inode for the supplied source object. The object's name
// must end with a slash unless this is the root directory, in which case it
// must be empty.
//
// If implicitDirs is set, LookUpChild will use ListObjects to find child
// directories that are "implicitly" defined by the existence of their own
// descendents. For example, if there is an object named "foo/bar/baz" and this
// is the directory "foo", a child directory named "bar" will be implied.
//
// The initial lookup count is one.
//
// REQUIRES: o != nil
// REQUIRES: o.Name == "" || o.Name[len(o.Name)-1] == '/'
func NewDirInode(
	bucket gcs.Bucket,
	id fuseops.InodeID,
	o *gcs.Object,
	implicitDirs bool) (d *DirInode) {
	if o.Name != "" && o.Name[len(o.Name)-1] != '/' {
		panic(fmt.Sprintf("Unexpected name: %s", o.Name))
	}

	// Set up the struct.
	d = &DirInode{
		bucket:       bucket,
		id:           id,
		implicitDirs: implicitDirs,
		src:          *o,
	}

	d.lc = lookupCount{
		count:   1,
		destroy: func() error { return nil },
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (d *DirInode) checkInvariants() {
	// INVARIANT: src.Name == "" || src.Name[len(name)-1] == '/'
	if !(d.src.Name == "" || d.src.Name[len(d.src.Name)-1] == '/') {
		panic(fmt.Sprintf("Unexpected name: %s", d.src.Name))
	}
}

func (d *DirInode) clobbered(ctx context.Context) (clobbered bool, err error) {
	// Special case: the root is never clobbered.
	if d.ID() == fuseops.RootInodeID {
		return
	}

	// Stat the backing object.
	req := &gcs.StatObjectRequest{
		Name: d.Name(),
	}

	o, err := d.bucket.StatObject(ctx, req)

	// "Not found" means clobbered.
	if _, ok := err.(*gcs.NotFoundError); ok {
		clobbered = true
		err = nil
		return
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	// We are clobbered if the generation number has changed.
	clobbered = o.Generation != d.SourceGeneration()

	return
}

func (d *DirInode) lookUpChildFile(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name)
	if err != nil {
		err = fmt.Errorf("statObjectMayNotExist: %v", err)
		return
	}

	return
}

func (d *DirInode) lookUpChildDir(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Stat the placeholder object.
	b.Add(func(ctx context.Context) (err error) {
		o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name+"/")
		if err != nil {
			err = fmt.Errorf("statObjectMayNotExist: %v", err)
			return
		}

		return
	})

	// If implicit directories are enabled, find out whether the child name is
	// implicitly defined.
	var implicitlyDefined bool
	if d.implicitDirs {
		b.Add(func(ctx context.Context) (err error) {
			implicitlyDefined, err = objectNamePrefixNonEmpty(
				ctx,
				d.bucket,
				d.Name()+name+"/")

			if err != nil {
				err = fmt.Errorf("objectNamePrefixNonEmpty: %v", err)
				return
			}

			return
		})
	}

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// If statting failed by the directory is implicitly defined, fake a source
	// object.
	if o == nil && implicitlyDefined {
		o = &gcs.Object{
			Name:       d.Name() + name + "/",
			Generation: ImplicitDirGen,
		}
	}

	return
}

// REQUIRES: strings.HasSuffix(name, ConflictingFileNameSuffix)
func (d *DirInode) lookUpConflicting(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	strippedName := strings.TrimSuffix(name, ConflictingFileNameSuffix)

	// In order to a marked name to be accepted, we require the conflicting
	// directory to exist.
	var dir *gcs.Object
	dir, err = d.lookUpChildDir(ctx, strippedName)
	if err != nil {
		err = fmt.Errorf("Looking up stripped name: %v", err)
		return
	}

	if dir == nil {
		err = fuse.ENOENT
		return
	}

	// The directory name exists. Find the conflicting file.
	o, err = d.lookUpChildFile(ctx, strippedName)
	if err != nil {
		return
	}

	if o == nil {
		err = fuse.ENOENT
		return
	}

	return
}

// List the supplied object name prefix to find out whether it is non-empty.
func objectNamePrefixNonEmpty(
	ctx context.Context,
	bucket gcs.Bucket,
	prefix string) (nonEmpty bool, err error) {
	req := &gcs.ListObjectsRequest{
		Prefix:     prefix,
		MaxResults: 1,
	}

	listing, err := bucket.ListObjects(ctx, req)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	nonEmpty = len(listing.Objects) != 0
	return
}

// Stat the object with the given name, returning (nil, nil) if the object
// doesn't exist rather than failing.
func statObjectMayNotExist(
	ctx context.Context,
	bucket gcs.Bucket,
	name string) (o *gcs.Object, err error) {
	// Call the bucket.
	req := &gcs.StatObjectRequest{
		Name: name,
	}

	o, err = bucket.StatObject(ctx, req)

	// Suppress "not found" errors.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
	}

	// Annotate others.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (d *DirInode) Lock() {
	// We don't require locks.
}

func (d *DirInode) Unlock() {
	// We don't require locks.
}

// Return the ID of this inode.
func (d *DirInode) ID() fuseops.InodeID {
	return d.id
}

// Return the full name of the directory object in GCS, including the trailing
// slash (e.g. "foo/bar/").
func (d *DirInode) Name() string {
	return d.src.Name
}

// Return the generation number from which this inode was branched.
func (d *DirInode) SourceGeneration() int64 {
	return d.src.Generation
}

// LOCKS_REQUIRED(d.mu)
func (d *DirInode) IncrementLookupCount() {
	d.lc.Inc()
}

// LOCKS_REQUIRED(d.mu)
func (d *DirInode) DecrementLookupCount(n uint64) (destroyed bool) {
	destroyed = d.lc.Dec(n)
	return
}

func (d *DirInode) Attributes(
	ctx context.Context) (attrs fuseops.InodeAttributes, err error) {
	// Find out whether the backing object has been clobbered in GCS.
	clobbered, err := d.clobbered(ctx)
	if err != nil {
		err = fmt.Errorf("clobbered: %v", err)
		return
	}

	// Set up basic attributes.
	attrs = fuseops.InodeAttributes{
		Mode: 0700 | os.ModeDir,
	}

	// Modify Nlink as appropriate.
	if !clobbered {
		attrs.Nlink = 1
	}

	return
}

// See notes on DirInode.LookUpChild.
const ImplicitDirGen int64 = -1

// A suffix that can be used to unambiguously tag a file system name.
// (Unambiguous because U+000A is not allowed in GCS object names.) This is
// used to refer to the file in a (file, directory) pair with conflicting
// object names.
//
// See also the notes on DirInode.LookUpChild.
const ConflictingFileNameSuffix = "\n"

// Look up the direct child with the given relative name, returning a record
// for the current object of that name in the GCS bucket. If both a file and a
// directory with the given name exist, the directory is preferred. Return
// fuse.ENOENT if neither is found.
//
// Special case: if the name ends in ConflictingFileNameSuffix, we strip the
// suffix, confirm that a conflicting directory exists, then return a record
// for the file.
//
// If this inode was created with implicitDirs is set, this method will use
// ListObjects to find child directories that are "implicitly" defined by the
// existence of their own descendents. For example, if there is an object named
// "foo/bar/baz" and this is the directory "foo", a child directory named "bar"
// will be implied. In this case, the child's generation will be the sentinel
// value ImplicitDirGen.
func (d *DirInode) LookUpChild(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Is this a conflict marker name?
	if strings.HasSuffix(name, ConflictingFileNameSuffix) {
		o, err = d.lookUpConflicting(ctx, name)
		return
	}

	// Stat the child as a file.
	var fileRecord *gcs.Object
	b.Add(func(ctx context.Context) (err error) {
		fileRecord, err = d.lookUpChildFile(ctx, name)
		return
	})

	// Stat the child as a directory.
	var dirRecord *gcs.Object
	b.Add(func(ctx context.Context) (err error) {
		dirRecord, err = d.lookUpChildDir(ctx, name)
		return
	})

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// Prefer directories over files.
	switch {
	case dirRecord != nil:
		o = dirRecord
	case fileRecord != nil:
		o = fileRecord
	default:
		err = fuse.ENOENT
	}

	return
}

// Read some number of entries from the directory, returning a continuation
// token that can be used to pick up the read operation where it left off.
// Supply the empty token on the first call.
//
// At the end of the directory, the returned continuation token will be empty.
// Otherwise it will be non-empty. There is no guarantee about the number of
// entries returned; it may be zero even with a non-empty continuation token.
//
// The contents of the Offset and Inode fields for returned entries is
// undefined.
func (d *DirInode) ReadEntries(
	ctx context.Context,
	tok string) (entries []fuseutil.Dirent, newTok string, err error) {
	// Ask the bucket to list some objects.
	req := &gcs.ListObjectsRequest{
		Delimiter:         "/",
		Prefix:            d.Name(),
		ContinuationToken: tok,
	}

	listing, err := d.bucket.ListObjects(ctx, req)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	// Convert objects to entries for files.
	for _, o := range listing.Objects {
		e := fuseutil.Dirent{
			Name: path.Base(o.Name),
			Type: fuseutil.DT_File,
		}

		entries = append(entries, e)
	}

	// Convert runs to entries for directories.
	for _, p := range listing.CollapsedRuns {
		e := fuseutil.Dirent{
			Name: path.Base(p),
			Type: fuseutil.DT_Directory,
		}

		entries = append(entries, e)
	}

	// Return an appropriate continuation token, if any.
	newTok = listing.ContinuationToken

	return
}
<|endoftext|>


<commit_before>package instances

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/accounts"
	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/globals"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/scheduler"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/cozy/cozy-stack/web/jsonapi"
	"github.com/labstack/echo"
)

type apiInstance struct {
	*instance.Instance
}

func (i *apiInstance) MarshalJSON() ([]byte, error) {
	return json.Marshal(i.Instance)
}

// Links is used to generate a JSON-API link for the instance
func (i *apiInstance) Links() *jsonapi.LinksList {
	return &jsonapi.LinksList{Self: "/instances/" + i.Instance.DocID}
}

// Relationships is used to generate the content relationship in JSON-API format
func (i *apiInstance) Relationships() jsonapi.RelationshipMap {
	return jsonapi.RelationshipMap{}
}

// Included is part of the jsonapi.Object interface
func (i *apiInstance) Included() []jsonapi.Object {
	return nil
}

func createHandler(c echo.Context) error {
	var diskQuota int64
	if c.QueryParam("DiskQuota") != "" {
		var err error
		diskQuota, err = strconv.ParseInt(c.QueryParam("DiskQuota"), 10, 64)
		if err != nil {
			return wrapError(err)
		}
	}
	var settings couchdb.JSONDoc
	settings.M = make(map[string]interface{})
	for _, setting := range strings.Split(c.QueryParam("Settings"), ",") {
		if parts := strings.SplitN(setting, ":", 2); len(parts) == 2 {
			settings.M[parts[0]] = parts[1]
		}
	}
	if tz := c.QueryParam("Timezone"); tz != "" {
		settings.M["tz"] = tz
	}
	if email := c.QueryParam("Email"); email != "" {
		settings.M["email"] = email
	}
	if name := c.QueryParam("PublicName"); name != "" {
		settings.M["public_name"] = name
	}

	fmt.Println(">>>>", c.QueryParam("SwiftCluster"))

	var swiftCluster int
	if cluster := c.QueryParam("SwiftCluster"); cluster != "" {
		var err error
		swiftCluster, err = strconv.Atoi(cluster)
		if err != nil {
			return wrapError(err)
		}
	}

	in, err := instance.Create(&instance.Options{
		Domain:       c.QueryParam("Domain"),
		Locale:       c.QueryParam("Locale"),
		DiskQuota:    diskQuota,
		Settings:     settings,
		SwiftCluster: swiftCluster,
		Apps:         utils.SplitTrimString(c.QueryParam("Apps"), ","),
		Dev:          (c.QueryParam("Dev") == "true"),
	})
	if err != nil {
		return wrapError(err)
	}
	in.OAuthSecret = nil
	in.SessionSecret = nil
	in.PassphraseHash = nil
	pass := c.QueryParam("Passphrase")
	if pass != "" {
		if err = in.RegisterPassphrase([]byte(pass), in.RegisterToken); err != nil {
			return err
		}
		// set the onboarding finished when specifying a passphrase. we totally
		// skip the onboarding in that case.
		in.OnboardingFinished = true
		if err = instance.Update(in); err != nil {
			return err
		}
	}
	return jsonapi.Data(c, http.StatusCreated, &apiInstance{in}, nil)
}

func showHandler(c echo.Context) error {
	domain := c.Param("domain")
	i, err := instance.Get(domain)
	if err != nil {
		return wrapError(err)
	}
	return jsonapi.Data(c, http.StatusOK, &apiInstance{i}, nil)
}

func modifyHandler(c echo.Context) error {
	domain := c.Param("domain")
	i, err := instance.Get(domain)
	if err != nil {
		return wrapError(err)
	}
	var shouldUpdate bool
	if quota := c.QueryParam("DiskQuota"); quota != "" {
		var diskQuota int64
		diskQuota, err = strconv.ParseInt(quota, 10, 64)
		if err != nil {
			return wrapError(err)
		}
		i.BytesDiskQuota = diskQuota
		shouldUpdate = true
	}
	if locale := c.QueryParam("Locale"); locale != "" {
		i.Locale = locale
		shouldUpdate = true
	}
	if onboardingFinished := c.QueryParam("OnboardingFinished"); onboardingFinished != "" {
		i.OnboardingFinished, err = strconv.ParseBool(onboardingFinished)
		if err != nil {
			return wrapError(err)
		}
		shouldUpdate = true
	}
	if shouldUpdate {
		if err = instance.Update(i); err != nil {
			return wrapError(err)
		}
	}
	if debug, err := strconv.ParseBool(c.QueryParam("Debug")); err == nil {
		if debug {
			err = logger.AddDebugDomain(domain)
		} else {
			err = logger.RemoveDebugDomain(domain)
		}
		if err != nil {
			return wrapError(err)
		}
	}
	return jsonapi.Data(c, http.StatusOK, &apiInstance{i}, nil)
}

func listHandler(c echo.Context) error {
	is, err := instance.List()
	if err != nil {
		if couchdb.IsNoDatabaseError(err) {
			return jsonapi.DataList(c, http.StatusOK, nil, nil)
		}
		return wrapError(err)
	}

	objs := make([]jsonapi.Object, len(is))
	for i, in := range is {
		in.OAuthSecret = nil
		in.SessionSecret = nil
		in.PassphraseHash = nil
		objs[i] = &apiInstance{in}
	}

	return jsonapi.DataList(c, http.StatusOK, objs, nil)
}

func deleteHandler(c echo.Context) error {
	domain := c.Param("domain")
	err := instance.Destroy(domain)
	if err != nil {
		return wrapError(err)
	}
	return c.NoContent(http.StatusNoContent)
}

func fsckHandler(c echo.Context) error {
	domain := c.Param("domain")
	i, err := instance.Get(domain)
	if err != nil {
		return wrapError(err)
	}
	prune, _ := strconv.ParseBool(c.QueryParam("Prune"))
	dryRun, _ := strconv.ParseBool(c.QueryParam("DryRun"))
	fs := i.VFS()
	logbook, err := fs.Fsck()
	if err != nil {
		return wrapError(err)
	}
	if prune {
		fs.FsckPrune(logbook, dryRun)
	}
	return c.JSON(http.StatusOK, logbook)
}

func rebuildRedis(c echo.Context) error {
	instances, err := instance.List()
	if err != nil {
		return wrapError(err)
	}
	for _, i := range instances {
		err = globals.GetScheduler().RebuildRedis(i.Domain)
		if err != nil {
			return wrapError(err)
		}
	}
	return c.NoContent(http.StatusNoContent)
}

func cleanOrphanAccounts(c echo.Context) error {
	type result struct {
		Result  string                  `json:"result"`
		Error   string                  `json:"error,omitempty"`
		Trigger *scheduler.TriggerInfos `json:"trigger,omitempty"`
	}

	dryRun, _ := strconv.ParseBool(c.QueryParam("DryRun"))
	results := make([]*result, 0)
	domain := c.Param("domain")
	db := couchdb.SimpleDatabasePrefix(domain)

	var as []*accounts.Account
	err := couchdb.GetAllDocs(db, consts.Accounts, nil, &as)
	if couchdb.IsNoDatabaseError(err) {
		return c.JSON(http.StatusOK, results)
	}
	if err != nil {
		return err
	}

	sched := globals.GetScheduler()
	ts, err := sched.GetAll(domain)
	if couchdb.IsNoDatabaseError(err) {
		return c.JSON(http.StatusOK, results)
	}
	if err != nil {
		return err
	}

	konnectors, err := apps.ListKonnectors(db)
	if couchdb.IsNoDatabaseError(err) {
		return c.JSON(http.StatusOK, results)
	}
	if err != nil {
		return err
	}

	triggersAccounts := make(map[string]struct{})

	for _, trigger := range ts {
		if trigger.Infos().WorkerType == "konnector" {
			var v struct {
				Account string `json:"account"`
			}
			if err = json.Unmarshal(trigger.Infos().Message, &v); err != nil {
				continue
			}
			if v.Account != "" {
				triggersAccounts[v.Account] = struct{}{}
			}
		}
	}

	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
	for _, account := range as {
		_, ok := triggersAccounts[account.ID()]
		if ok {
			continue
		}

		var konnectorFound bool
		for _, k := range konnectors {
			if k.Slug() == account.AccountType {
				konnectorFound = true
				break
			}
		}

		if !konnectorFound {
			continue
		}

		msg, _ := json.Marshal(struct {
			Account      string `json:"account"`
			Konnector    string `json:"konnector"`
			FolderToSave string `json:"folder_to_save"`
		}{
			Account:   account.ID(),
			Konnector: account.AccountType,
		})

		var args string
		{
			d := rng.Intn(7) + 1
			h := rng.Intn(6) // during the night, between 0 and 5
			m := rng.Intn(60)
			args = fmt.Sprintf("0 %d %d * * %d", m, h, d)
		}

		var r result
		infos := &scheduler.TriggerInfos{
			WorkerType: "konnector",
			Domain:     domain,
			Type:       "@cron",
			Arguments:  args,
			Message:    jobs.Message(msg),
		}
		r.Trigger = infos
		t, err := scheduler.NewTrigger(infos)
		if err != nil {
			r.Result = "failed"
			r.Error = err.Error()
			continue
		}
		if !dryRun {
			err = sched.Add(t)
		} else {
			err = nil
		}
		if err != nil {
			r.Result = "failed"
			r.Error = err.Error()
		} else {
			r.Result = "created"
		}
		results = append(results, &r)
	}

	return c.JSON(http.StatusOK, results)
}

func wrapError(err error) error {
	switch err {
	case instance.ErrNotFound:
		return jsonapi.NotFound(err)
	case instance.ErrExists:
		return jsonapi.Conflict(err)
	case instance.ErrIllegalDomain:
		return jsonapi.InvalidParameter("domain", err)
	case instance.ErrMissingToken:
		return jsonapi.BadRequest(err)
	case instance.ErrInvalidToken:
		return jsonapi.BadRequest(err)
	case instance.ErrMissingPassphrase:
		return jsonapi.BadRequest(err)
	case instance.ErrInvalidPassphrase:
		return jsonapi.BadRequest(err)
	}
	return err
}

// Routes sets the routing for the instances service
func Routes(router *echo.Group) {
	router.GET("", listHandler)
	router.POST("", createHandler)
	router.GET("/:domain", showHandler)
	router.PATCH("/:domain", modifyHandler)
	router.DELETE("/:domain", deleteHandler)
	router.GET("/:domain/fsck", fsckHandler)
	router.POST("/updates", updatesHandler)
	router.POST("/token", createToken)
	router.POST("/oauth_client", registerClient)
	router.POST("/:domain/export", exporter)
	router.POST("/:domain/import", importer)
	router.POST("/:domain/orphan_accounts", cleanOrphanAccounts)
	router.POST("/redis", rebuildRedis)
}
<commit_msg>Remove fmt.Println<commit_after>package instances

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/accounts"
	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/globals"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/scheduler"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/cozy/cozy-stack/web/jsonapi"
	"github.com/labstack/echo"
)

type apiInstance struct {
	*instance.Instance
}

func (i *apiInstance) MarshalJSON() ([]byte, error) {
	return json.Marshal(i.Instance)
}

// Links is used to generate a JSON-API link for the instance
func (i *apiInstance) Links() *jsonapi.LinksList {
	return &jsonapi.LinksList{Self: "/instances/" + i.Instance.DocID}
}

// Relationships is used to generate the content relationship in JSON-API format
func (i *apiInstance) Relationships() jsonapi.RelationshipMap {
	return jsonapi.RelationshipMap{}
}

// Included is part of the jsonapi.Object interface
func (i *apiInstance) Included() []jsonapi.Object {
	return nil
}

func createHandler(c echo.Context) error {
	var diskQuota int64
	if c.QueryParam("DiskQuota") != "" {
		var err error
		diskQuota, err = strconv.ParseInt(c.QueryParam("DiskQuota"), 10, 64)
		if err != nil {
			return wrapError(err)
		}
	}
	var settings couchdb.JSONDoc
	settings.M = make(map[string]interface{})
	for _, setting := range strings.Split(c.QueryParam("Settings"), ",") {
		if parts := strings.SplitN(setting, ":", 2); len(parts) == 2 {
			settings.M[parts[0]] = parts[1]
		}
	}
	if tz := c.QueryParam("Timezone"); tz != "" {
		settings.M["tz"] = tz
	}
	if email := c.QueryParam("Email"); email != "" {
		settings.M["email"] = email
	}
	if name := c.QueryParam("PublicName"); name != "" {
		settings.M["public_name"] = name
	}

	var swiftCluster int
	if cluster := c.QueryParam("SwiftCluster"); cluster != "" {
		var err error
		swiftCluster, err = strconv.Atoi(cluster)
		if err != nil {
			return wrapError(err)
		}
	}

	in, err := instance.Create(&instance.Options{
		Domain:       c.QueryParam("Domain"),
		Locale:       c.QueryParam("Locale"),
		DiskQuota:    diskQuota,
		Settings:     settings,
		SwiftCluster: swiftCluster,
		Apps:         utils.SplitTrimString(c.QueryParam("Apps"), ","),
		Dev:          (c.QueryParam("Dev") == "true"),
	})
	if err != nil {
		return wrapError(err)
	}
	in.OAuthSecret = nil
	in.SessionSecret = nil
	in.PassphraseHash = nil
	pass := c.QueryParam("Passphrase")
	if pass != "" {
		if err = in.RegisterPassphrase([]byte(pass), in.RegisterToken); err != nil {
			return err
		}
		// set the onboarding finished when specifying a passphrase. we totally
		// skip the onboarding in that case.
		in.OnboardingFinished = true
		if err = instance.Update(in); err != nil {
			return err
		}
	}
	return jsonapi.Data(c, http.StatusCreated, &apiInstance{in}, nil)
}

func showHandler(c echo.Context) error {
	domain := c.Param("domain")
	i, err := instance.Get(domain)
	if err != nil {
		return wrapError(err)
	}
	return jsonapi.Data(c, http.StatusOK, &apiInstance{i}, nil)
}

func modifyHandler(c echo.Context) error {
	domain := c.Param("domain")
	i, err := instance.Get(domain)
	if err != nil {
		return wrapError(err)
	}
	var shouldUpdate bool
	if quota := c.QueryParam("DiskQuota"); quota != "" {
		var diskQuota int64
		diskQuota, err = strconv.ParseInt(quota, 10, 64)
		if err != nil {
			return wrapError(err)
		}
		i.BytesDiskQuota = diskQuota
		shouldUpdate = true
	}
	if locale := c.QueryParam("Locale"); locale != "" {
		i.Locale = locale
		shouldUpdate = true
	}
	if onboardingFinished := c.QueryParam("OnboardingFinished"); onboardingFinished != "" {
		i.OnboardingFinished, err = strconv.ParseBool(onboardingFinished)
		if err != nil {
			return wrapError(err)
		}
		shouldUpdate = true
	}
	if shouldUpdate {
		if err = instance.Update(i); err != nil {
			return wrapError(err)
		}
	}
	if debug, err := strconv.ParseBool(c.QueryParam("Debug")); err == nil {
		if debug {
			err = logger.AddDebugDomain(domain)
		} else {
			err = logger.RemoveDebugDomain(domain)
		}
		if err != nil {
			return wrapError(err)
		}
	}
	return jsonapi.Data(c, http.StatusOK, &apiInstance{i}, nil)
}

func listHandler(c echo.Context) error {
	is, err := instance.List()
	if err != nil {
		if couchdb.IsNoDatabaseError(err) {
			return jsonapi.DataList(c, http.StatusOK, nil, nil)
		}
		return wrapError(err)
	}

	objs := make([]jsonapi.Object, len(is))
	for i, in := range is {
		in.OAuthSecret = nil
		in.SessionSecret = nil
		in.PassphraseHash = nil
		objs[i] = &apiInstance{in}
	}

	return jsonapi.DataList(c, http.StatusOK, objs, nil)
}

func deleteHandler(c echo.Context) error {
	domain := c.Param("domain")
	err := instance.Destroy(domain)
	if err != nil {
		return wrapError(err)
	}
	return c.NoContent(http.StatusNoContent)
}

func fsckHandler(c echo.Context) error {
	domain := c.Param("domain")
	i, err := instance.Get(domain)
	if err != nil {
		return wrapError(err)
	}
	prune, _ := strconv.ParseBool(c.QueryParam("Prune"))
	dryRun, _ := strconv.ParseBool(c.QueryParam("DryRun"))
	fs := i.VFS()
	logbook, err := fs.Fsck()
	if err != nil {
		return wrapError(err)
	}
	if prune {
		fs.FsckPrune(logbook, dryRun)
	}
	return c.JSON(http.StatusOK, logbook)
}

func rebuildRedis(c echo.Context) error {
	instances, err := instance.List()
	if err != nil {
		return wrapError(err)
	}
	for _, i := range instances {
		err = globals.GetScheduler().RebuildRedis(i.Domain)
		if err != nil {
			return wrapError(err)
		}
	}
	return c.NoContent(http.StatusNoContent)
}

func cleanOrphanAccounts(c echo.Context) error {
	type result struct {
		Result  string                  `json:"result"`
		Error   string                  `json:"error,omitempty"`
		Trigger *scheduler.TriggerInfos `json:"trigger,omitempty"`
	}

	dryRun, _ := strconv.ParseBool(c.QueryParam("DryRun"))
	results := make([]*result, 0)
	domain := c.Param("domain")
	db := couchdb.SimpleDatabasePrefix(domain)

	var as []*accounts.Account
	err := couchdb.GetAllDocs(db, consts.Accounts, nil, &as)
	if couchdb.IsNoDatabaseError(err) {
		return c.JSON(http.StatusOK, results)
	}
	if err != nil {
		return err
	}

	sched := globals.GetScheduler()
	ts, err := sched.GetAll(domain)
	if couchdb.IsNoDatabaseError(err) {
		return c.JSON(http.StatusOK, results)
	}
	if err != nil {
		return err
	}

	konnectors, err := apps.ListKonnectors(db)
	if couchdb.IsNoDatabaseError(err) {
		return c.JSON(http.StatusOK, results)
	}
	if err != nil {
		return err
	}

	triggersAccounts := make(map[string]struct{})

	for _, trigger := range ts {
		if trigger.Infos().WorkerType == "konnector" {
			var v struct {
				Account string `json:"account"`
			}
			if err = json.Unmarshal(trigger.Infos().Message, &v); err != nil {
				continue
			}
			if v.Account != "" {
				triggersAccounts[v.Account] = struct{}{}
			}
		}
	}

	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
	for _, account := range as {
		_, ok := triggersAccounts[account.ID()]
		if ok {
			continue
		}

		var konnectorFound bool
		for _, k := range konnectors {
			if k.Slug() == account.AccountType {
				konnectorFound = true
				break
			}
		}

		if !konnectorFound {
			continue
		}

		msg, _ := json.Marshal(struct {
			Account      string `json:"account"`
			Konnector    string `json:"konnector"`
			FolderToSave string `json:"folder_to_save"`
		}{
			Account:   account.ID(),
			Konnector: account.AccountType,
		})

		var args string
		{
			d := rng.Intn(7) + 1
			h := rng.Intn(6) // during the night, between 0 and 5
			m := rng.Intn(60)
			args = fmt.Sprintf("0 %d %d * * %d", m, h, d)
		}

		var r result
		infos := &scheduler.TriggerInfos{
			WorkerType: "konnector",
			Domain:     domain,
			Type:       "@cron",
			Arguments:  args,
			Message:    jobs.Message(msg),
		}
		r.Trigger = infos
		t, err := scheduler.NewTrigger(infos)
		if err != nil {
			r.Result = "failed"
			r.Error = err.Error()
			continue
		}
		if !dryRun {
			err = sched.Add(t)
		} else {
			err = nil
		}
		if err != nil {
			r.Result = "failed"
			r.Error = err.Error()
		} else {
			r.Result = "created"
		}
		results = append(results, &r)
	}

	return c.JSON(http.StatusOK, results)
}

func wrapError(err error) error {
	switch err {
	case instance.ErrNotFound:
		return jsonapi.NotFound(err)
	case instance.ErrExists:
		return jsonapi.Conflict(err)
	case instance.ErrIllegalDomain:
		return jsonapi.InvalidParameter("domain", err)
	case instance.ErrMissingToken:
		return jsonapi.BadRequest(err)
	case instance.ErrInvalidToken:
		return jsonapi.BadRequest(err)
	case instance.ErrMissingPassphrase:
		return jsonapi.BadRequest(err)
	case instance.ErrInvalidPassphrase:
		return jsonapi.BadRequest(err)
	}
	return err
}

// Routes sets the routing for the instances service
func Routes(router *echo.Group) {
	router.GET("", listHandler)
	router.POST("", createHandler)
	router.GET("/:domain", showHandler)
	router.PATCH("/:domain", modifyHandler)
	router.DELETE("/:domain", deleteHandler)
	router.GET("/:domain/fsck", fsckHandler)
	router.POST("/updates", updatesHandler)
	router.POST("/token", createToken)
	router.POST("/oauth_client", registerClient)
	router.POST("/:domain/export", exporter)
	router.POST("/:domain/import", importer)
	router.POST("/:domain/orphan_accounts", cleanOrphanAccounts)
	router.POST("/redis", rebuildRedis)
}
<|endoftext|>


<commit_before>package bit

type Bit byte

func (b *Bit) Flip() {
	(*b) ^= 0x01
}

type Bits []Bit

func toBits(b byte) Bits {
	var o = make(Bits, 8)
	for bit, mask := 0, byte(128); bit < 8; bit, mask = bit+1, mask>>1 {
		if b&mask != 0 {
			o[bit] = 1
		}
	}
	return o
}

func NewBits(bytes []byte) Bits {
	var l = len(bytes)
	var o = make(Bits, 0)
	for i := 0; i < l; i++ {
		o = append(o, toBits(bytes[i])...)
	}
	return o
}

func (bits *Bits) Bytes() []byte {
	var l = len(*bits)
	var o = make([]byte, (l+7)/8)
	for i, b := range *bits {
		if b == 0x01 {
			o[i/8] |= (1 << byte(7-(i%8)))
		}
	}
	return o
}

func (bits *Bits) String() string {
	var s = ""
	for _, b := range *bits {
		if b == 0x01 {
			s += "1"
		} else {
			s += "0"
		}
	}
	return s
}
<commit_msg>bit: added .Equal() and .Len()<commit_after>package bit

type Bit byte

func (b *Bit) Flip() {
	(*b) ^= 0x01
}

type Bits []Bit

func toBits(b byte) Bits {
	var o = make(Bits, 8)
	for bit, mask := 0, byte(128); bit < 8; bit, mask = bit+1, mask>>1 {
		if b&mask != 0 {
			o[bit] = 1
		}
	}
	return o
}

func NewBits(bytes []byte) Bits {
	var l = len(bytes)
	var o = make(Bits, 0)
	for i := 0; i < l; i++ {
		o = append(o, toBits(bytes[i])...)
	}
	return o
}

func (bits *Bits) Bytes() []byte {
	var l = len(*bits)
	var o = make([]byte, (l+7)/8)
	for i, b := range *bits {
		if b == 0x01 {
			o[i/8] |= (1 << byte(7-(i%8)))
		}
	}
	return o
}

func (bits Bits) Equal(other Bits) bool {
	var l = bits.Len()
	if l != other.Len() {
		return false
	}
	for i := 0; i < l; i++ {
		if bits[i] != other[i] {
			return false
		}
	}
	return true
}

func (bits Bits) Len() int {
	return len(bits)
}

func (bits Bits) String() string {
	var s = ""
	for _, b := range bits {
		if b == 0x01 {
			s += "1"
		} else {
			s += "0"
		}
	}
	return s
}
<|endoftext|>


<commit_before>package layers

import tl "github.com/JoelOtter/termloop"

// Manager manages collection of named Drawable objects
type Manager struct {
	keys   []string
	layers map[string]tl.Drawable
}

// NewManager returns new Manager
func NewManager() *Manager {
	return &Manager{
		keys:   make([]string, 0),
		layers: make(map[string]tl.Drawable),
	}
}

// Add adds named Drawable to the manager
func (m *Manager) Add(name string, d tl.Drawable) {
	m.keys = append(m.keys, name)
	m.layers[name] = d
}

// Remove removes named drawable from manager
func (m *Manager) Remove(name string) bool {
	i := m.keyIndex(name)
	if i != -1 {
		m.keys = append(m.keys[:i], m.keys[i+1:]...)
		delete(m.layers, name)
		return true
	}
	return false
}

// Len returns the number of elements in the manager
func (m *Manager) Len() int {
	return len(m.layers)
}

// Tick processes termloop Tick
func (m *Manager) Tick(e tl.Event) {
	for _, key := range m.keys {
		m.layers[key].Tick(e)
	}
}

// Draw processes termloop Draw
func (m *Manager) Draw(s *tl.Screen) {
	for _, key := range m.keys {
		m.layers[key].Draw(s)
	}
}

// SetOrder sets the order in which the objects will be drawn
func (m *Manager) SetOrder(keys []string) int {
	m.keys = m.keys[:0]
	i := 0
	for _, key := range keys {
		if _, ok := m.layers[key]; ok {
			m.keys = append(m.keys, key)
			i++
		}
	}
	return i
}

func (m *Manager) keyIndex(name string) int {
	for i, key := range m.keys {
		if key == name {
			return i
		}
	}
	return -1
}
<commit_msg>Added SetOrder function<commit_after>package layers

import tl "github.com/JoelOtter/termloop"

// Manager manages collection of named Drawable objects
type Manager struct {
	keys   []string
	layers map[string]tl.Drawable
}

// NewManager returns new Manager
func NewManager() *Manager {
	return &Manager{
		keys:   make([]string, 0),
		layers: make(map[string]tl.Drawable),
	}
}

// Add adds named Drawable to the manager
func (m *Manager) Add(name string, d tl.Drawable) {
	m.keys = append(m.keys, name)
	m.layers[name] = d
}

// Remove removes named drawable from manager
func (m *Manager) Remove(name string) bool {
	i := m.keyIndex(name)
	if i != -1 {
		m.keys = append(m.keys[:i], m.keys[i+1:]...)
		delete(m.layers, name)
		return true
	}
	return false
}

// Len returns the number of elements in the manager
func (m *Manager) Len() int {
	return len(m.layers)
}

// Tick processes termloop Tick
func (m *Manager) Tick(e tl.Event) {
	for _, key := range m.keys {
		m.layers[key].Tick(e)
	}
}

// Draw processes termloop Draw
func (m *Manager) Draw(s *tl.Screen) {
	for _, key := range m.keys {
		m.layers[key].Draw(s)
	}
}

// SetOrder sets the order in which the objects will be drawn and returns the
// number of found objects
func (m *Manager) SetOrder(keys []string) int {
	m.keys = m.keys[:0]
	i := 0
	for _, key := range keys {
		if _, ok := m.layers[key]; ok {
			m.keys = append(m.keys, key)
			i++
		}
	}
	return i
}

func (m *Manager) keyIndex(name string) int {
	for i, key := range m.keys {
		if key == name {
			return i
		}
	}
	return -1
}
<|endoftext|>


<commit_before>package dependency

import (
	"fmt"
	"github.com/deckarep/golang-set"
)

type Dependency struct {
	Name     string
	Version  string
	Requires map[string]string
}

func GetPackageNames(d Dependency) mapset.Set {
	packages := mapset.NewSet()

	for packageName, _ := range d.Requires {
		packages.Add(packageName)
	}

	return packages
}

type DependencyFetcher interface {
	Get(dependencyName string) (map[string]Dependency, error)
}

type DependencyRepo struct {
	DependencyNames mapset.Set
	Dependencies    map[string]map[string]Dependency
	Fetcher         DependencyFetcher
}

func (r DependencyRepo) GetAll(dependencies mapset.Set) {

	f := dependencies.Intersect(r.DependencyNames)

	if f.Cardinality() == dependencies.Cardinality() {
		return
	}

	subDependencies := mapset.NewSet()
	for _, nameI := range dependencies.ToSlice() {
		name := fmt.Sprintf("%s", nameI)
		r.DependencyNames.Add(name)
		dm, err := r.Fetcher.Get(name)

		if err != nil {
			panic(err)
		}

		for _, sd := range dm {
			for packageName, _ := range sd.Requires {
				subDependencies.Add(packageName)
			}
		}

	}
	r.GetAll(subDependencies)
}

func GetNewRepo(fetcher DependencyFetcher) DependencyRepo {
	dr := DependencyRepo{

		DependencyNames: mapset.NewSet(),
		Dependencies:    map[string]map[string]Dependency{},
		Fetcher:         fetcher,
	}

	return dr
}
<commit_msg>Assign dependency to Repo member<commit_after>package dependency

import (
	"fmt"
	"github.com/deckarep/golang-set"
)

type Dependency struct {
	Name     string
	Version  string
	Requires map[string]string
}

func GetPackageNames(d Dependency) mapset.Set {
	packages := mapset.NewSet()

	for packageName, _ := range d.Requires {
		packages.Add(packageName)
	}

	return packages
}

type DependencyFetcher interface {
	Get(dependencyName string) (map[string]Dependency, error)
}

type DependencyRepo struct {
	DependencyNames mapset.Set
	Dependencies    map[string]map[string]Dependency
	Fetcher         DependencyFetcher
}

func (r DependencyRepo) GetAll(dependencies mapset.Set) {

	// check to see if there is 100% intersect if so bail
	f := dependencies.Intersect(r.DependencyNames)
	if f.Cardinality() == dependencies.Cardinality() {
		return
	}

	subDependencies := mapset.NewSet()
	for _, nameI := range dependencies.ToSlice() {
		name := fmt.Sprintf("%s", nameI)
		r.DependencyNames.Add(name)
		dm, err := r.Fetcher.Get(name)

		if err != nil {
			panic(err)
		}
		r.Dependencies[name] = dm
		for _, sd := range dm {
			for packageName, _ := range sd.Requires {
				subDependencies.Add(packageName)
			}
		}

	}
	r.GetAll(subDependencies)
}

func GetNewRepo(fetcher DependencyFetcher) DependencyRepo {
	dr := DependencyRepo{

		DependencyNames: mapset.NewSet(),
		Dependencies:    map[string]map[string]Dependency{},
		Fetcher:         fetcher,
	}

	return dr
}
<|endoftext|>


<commit_before>// +build freebsd netbsd openbsd dragonfly darwin

package immortal

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"testing"
)

func TestNewScanDir(t *testing.T) {
	_, err := NewScanDir("/tmp")
	if err != nil {
		t.Error(err)
	}
}

func TestNewScanDirNonexistent(t *testing.T) {
	_, err := NewScanDir("/dev/null/non-existent")
	if err == nil {
		t.Error("Expecting error")
	}
}

type mockController struct {
	t      *testing.T
	i, j   int
	expect []struct {
		socket      string
		signal      []string
		signalErr   bool
		cmd, runErr string
	}
	status chan string
}

func (mc *mockController) GetStatus(socket string) (*Status, error) {
	status := &Status{}
	return status, nil
}

func (mc *mockController) SendSignal(socket, signal string) (*SignalResponse, error) {
	defer func() {
		mc.status <- signal
	}()
	mc.j++
	expect(mc.t, mc.expect[mc.i].socket, socket)
	expect(mc.t, mc.expect[mc.i].signal[mc.j], signal)
	if mc.expect[mc.i].signalErr {
		return nil, fmt.Errorf("error")
	}
	return nil, nil
}

func (mc *mockController) FindServices(dir string) ([]*ServiceStatus, error) {
	return nil, nil
}

func (mc *mockController) PurgeServices(dir string) error {
	return nil
}

func (mc *mockController) Run(command string) ([]byte, error) {
	defer func() {
		mc.status <- "Run"
	}()
	cmd := fmt.Sprintf("immortal -c %s/run.yml -ctl run", mc.expect[mc.i].cmd)
	expect(mc.t, cmd, command)
	if mc.expect[mc.i].runErr != "" {
		return nil, fmt.Errorf("%s\n", mc.expect[mc.i].runErr)
	}
	return []byte(fmt.Sprintf("started %d", mc.i)), nil
}

func TestScanner(t *testing.T) {
	dir, err := ioutil.TempDir("", "scaner")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up
	s, err := NewScanDir(dir)
	if err != nil {
		t.Fatal(err)
	}
	ctl := &mockController{
		t: t,
		i: 0,  //  number of scan calls
		j: -1, // number of calls within the same Scan
		expect: []struct {
			socket      string
			signal      []string
			signalErr   bool
			cmd, runErr string
		}{
			{"/var/run/immortal/run/immortal.sock", []string{}, true, s.scandir, ""},
			{"/var/run/immortal/run/immortal.sock", []string{"halt", "start"}, true, s.scandir, "return error 1"},
			{"/var/run/immortal/run/immortal.sock", []string{"halt"}, false, "", ""},
			{"/var/run/immortal/run/immortal.sock", []string{"start"}, true, s.scandir, "can't start"},
			{"/var/run/immortal/run/immortal.sock", []string{"start"}, false, "", ""},
			{"/var/run/immortal/run/immortal.sock", []string{}, false, "", ""},
			{"/var/run/immortal/run/immortal.sock", []string{}, false, "", ""},
			{"/var/run/immortal/run/immortal.sock", []string{"start"}, false, "", ""},
			{"/var/run/immortal/run/immortal.sock", []string{"halt"}, false, "", ""},
		},
		status: make(chan string, 1),
	}
	// start scanner loop
	go s.Start(ctl)
	if err = ioutil.WriteFile(filepath.Join(dir, "run.yml"), []byte("stage 0"), 0644); err != nil {
		t.Fatal(err)
	}
	var status string
	status = <-ctl.status
	expect(t, status, "Run")
	expect(t, "2bf41d668dd3b0909d58f982aff35a25", s.services["run"])
	ctl.i++
	ctl.j = -1

	// change service contents, a restart (exit, start) is expected
	if err = ioutil.WriteFile(filepath.Join(dir, "run.yml"), []byte("stage 1"), 0644); err != nil {
		t.Fatal(err)
	}
	status = <-ctl.status
	expect(t, status, "halt")
	status = <-ctl.status
	expect(t, status, "start")
	ctl.i++
	ctl.j = -1
}
<commit_msg>	modified:   scandir_test.go<commit_after>// +build freebsd netbsd openbsd dragonfly darwin

package immortal

import (
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"testing"
)

func TestNewScanDir(t *testing.T) {
	_, err := NewScanDir("/tmp")
	if err != nil {
		t.Error(err)
	}
}

func TestNewScanDirNonexistent(t *testing.T) {
	_, err := NewScanDir("/dev/null/non-existent")
	if err == nil {
		t.Error("Expecting error")
	}
}

type mockController struct {
	t      *testing.T
	i, j   int
	expect []struct {
		socket      string
		signal      []string
		signalErr   bool
		cmd, runErr string
	}
	status chan string
}

func (mc *mockController) GetStatus(socket string) (*Status, error) {
	status := &Status{}
	return status, nil
}

func (mc *mockController) SendSignal(socket, signal string) (*SignalResponse, error) {
	defer func() {
		mc.status <- signal
	}()
	return nil, nil
}

func (mc *mockController) FindServices(dir string) ([]*ServiceStatus, error) {
	return nil, nil
}

func (mc *mockController) PurgeServices(dir string) error {
	return nil
}

func (mc *mockController) Run(command string) ([]byte, error) {
	defer func() {
		mc.status <- "Run"
	}()
	return nil, nil
}

func TestScanner(t *testing.T) {
	dir, err := ioutil.TempDir("", "scaner")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up
	s, err := NewScanDir(dir)
	if err != nil {
		t.Fatal(err)
	}
	ctl := &mockController{
		status: make(chan string),
	}
	if err = ioutil.WriteFile(filepath.Join(dir, "run.yml"), []byte("stage 0"), 0644); err != nil {
		t.Fatal(err)
	}

	// start scanner loop
	go s.Start(ctl)

	var status string
	status = <-ctl.status
	expect(t, status, "Run")
	expect(t, "2bf41d668dd3b0909d58f982aff35a25", s.services["run"])

	// change service contents, a restart (exit, start) is expected
	if err = ioutil.WriteFile(filepath.Join(dir, "run.yml"), []byte("stage 1"), 0644); err != nil {
		t.Fatal(err)
	}
	status = <-ctl.status
	expect(t, status, "halt")
	status = <-ctl.status
	expect(t, status, "start")

	// remove service, exit
	if err := os.Remove(filepath.Join(dir, "run.yml")); err != nil {
		t.Fatal(err)
	}

}
<|endoftext|>


<commit_before>// Copyright 2014 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"os"
	"runtime"
	"syscall"

	"github.com/appc/cni/pkg/ip"
	"github.com/appc/cni/pkg/ipam"
	"github.com/appc/cni/pkg/ns"
	"github.com/appc/cni/pkg/skel"
	"github.com/appc/cni/pkg/types"
	"github.com/appc/cni/pkg/utils"
	"github.com/vishvananda/netlink"
)

const defaultBrName = "cni0"

type NetConf struct {
	types.NetConf
	BrName string `json:"bridge"`
	IsGW   bool   `json:"isGateway"`
	IPMasq bool   `json:"ipMasq"`
	MTU    int    `json:"mtu"`
}

func init() {
	// this ensures that main runs only on main thread (thread group leader).
	// since namespace ops (unshare, setns) are done for a single thread, we
	// must ensure that the goroutine does not jump from OS thread to thread
	runtime.LockOSThread()
}

func loadNetConf(bytes []byte) (*NetConf, error) {
	n := &NetConf{
		BrName: defaultBrName,
	}
	if err := json.Unmarshal(bytes, n); err != nil {
		return nil, fmt.Errorf("failed to load netconf: %v", err)
	}
	return n, nil
}

func ensureBridgeAddr(br *netlink.Bridge, ipn *net.IPNet) error {
	addrs, err := netlink.AddrList(br, syscall.AF_INET)
	if err != nil && err != syscall.ENOENT {
		return fmt.Errorf("could not get list of IP addresses: %v", err)
	}

	// if there're no addresses on the bridge, it's ok -- we'll add one
	if len(addrs) > 0 {
		ipnStr := ipn.String()
		for _, a := range addrs {
			// string comp is actually easiest for doing IPNet comps
			if a.IPNet.String() == ipnStr {
				return nil
			}
		}
		return fmt.Errorf("%q already has an IP address different from %v", br.Name, ipn.String())
	}

	addr := &netlink.Addr{IPNet: ipn, Label: ""}
	if err := netlink.AddrAdd(br, addr); err != nil {
		return fmt.Errorf("could not add IP address to %q: %v", br.Name, err)
	}
	return nil
}

func bridgeByName(name string) (*netlink.Bridge, error) {
	l, err := netlink.LinkByName(name)
	if err != nil {
		return nil, fmt.Errorf("could not lookup %q: %v", name, err)
	}
	br, ok := l.(*netlink.Bridge)
	if !ok {
		return nil, fmt.Errorf("%q already exists but is not a bridge", name)
	}
	return br, nil
}

func ensureBridge(brName string, mtu int) (*netlink.Bridge, error) {
	br := &netlink.Bridge{
		LinkAttrs: netlink.LinkAttrs{
			Name: brName,
			MTU:  mtu,
		},
	}

	if err := netlink.LinkAdd(br); err != nil {
		if err != syscall.EEXIST {
			return nil, fmt.Errorf("could not add %q: %v", brName, err)
		}

		// it's ok if the device already exists as long as config is similar
		br, err = bridgeByName(brName)
		if err != nil {
			return nil, err
		}
	}

	if err := netlink.LinkSetUp(br); err != nil {
		return nil, err
	}

	return br, nil
}

func setupVeth(netns string, br *netlink.Bridge, ifName string, mtu int) error {
	var hostVethName string

	err := ns.WithNetNSPath(netns, false, func(hostNS *os.File) error {
		// create the veth pair in the container and move host end into host netns
		hostVeth, _, err := ip.SetupVeth(ifName, mtu, hostNS)
		if err != nil {
			return err
		}

		hostVethName = hostVeth.Attrs().Name
		return nil
	})
	if err != nil {
		return err
	}

	// need to lookup hostVeth again as its index has changed during ns move
	hostVeth, err := netlink.LinkByName(hostVethName)
	if err != nil {
		return fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
	}

	// connect host veth end to the bridge
	if err = netlink.LinkSetMaster(hostVeth, br); err != nil {
		return fmt.Errorf("failed to connect %q to bridge %v: %v", hostVethName, br.Attrs().Name, err)
	}

	return nil
}

func calcGatewayIP(ipn *net.IPNet) net.IP {
	nid := ipn.IP.Mask(ipn.Mask)
	return ip.NextIP(nid)
}

func setupBridge(n *NetConf) (*netlink.Bridge, error) {
	// create bridge if necessary
	br, err := ensureBridge(n.BrName, n.MTU)
	if err != nil {
		return nil, fmt.Errorf("failed to create bridge %q: %v", n.BrName, err)
	}

	return br, nil
}

func cmdAdd(args *skel.CmdArgs) error {
	n, err := loadNetConf(args.StdinData)
	if err != nil {
		return err
	}

	br, err := setupBridge(n)
	if err != nil {
		return err
	}

	if err = setupVeth(args.Netns, br, args.IfName, n.MTU); err != nil {
		return err
	}

	// run the IPAM plugin and get back the config to apply
	result, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
	if err != nil {
		return err
	}

	if result.IP4 == nil {
		return errors.New("IPAM plugin returned missing IPv4 config")
	}

	if result.IP4.Gateway == nil && n.IsGW {
		result.IP4.Gateway = calcGatewayIP(&result.IP4.IP)
	}

	err = ns.WithNetNSPath(args.Netns, false, func(hostNS *os.File) error {
		return ipam.ConfigureIface(args.IfName, result)
	})
	if err != nil {
		return err
	}

	if n.IsGW {
		gwn := &net.IPNet{
			IP:   result.IP4.Gateway,
			Mask: result.IP4.IP.Mask,
		}

		if err = ensureBridgeAddr(br, gwn); err != nil {
			return err
		}

		if err := ip.EnableIP4Forward(); err != nil {
			return fmt.Errorf("failed to enable forwarding: %v", err)
		}
	}

	if n.IPMasq {
		chain := utils.FormatChainName(n.Name, args.ContainerID)
		comment := utils.FormatComment(n.Name, args.ContainerID)
		if err = ip.SetupIPMasq(ip.Network(&result.IP4.IP), chain, comment); err != nil {
			return err
		}
	}

	result.DNS = n.DNS
	return result.Print()
}

func cmdDel(args *skel.CmdArgs) error {
	n, err := loadNetConf(args.StdinData)
	if err != nil {
		return err
	}

	err = ipam.ExecDel(n.IPAM.Type, args.StdinData)
	if err != nil {
		return err
	}

	return ns.WithNetNSPath(args.Netns, false, func(hostNS *os.File) error {
		return ip.DelLinkByName(args.IfName)
	})
}

func main() {
	skel.PluginMain(cmdAdd, cmdDel)
}
<commit_msg>plugins/bridge: clean masquerading rules<commit_after>// Copyright 2014 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"os"
	"runtime"
	"syscall"

	"github.com/appc/cni/pkg/ip"
	"github.com/appc/cni/pkg/ipam"
	"github.com/appc/cni/pkg/ns"
	"github.com/appc/cni/pkg/skel"
	"github.com/appc/cni/pkg/types"
	"github.com/appc/cni/pkg/utils"
	"github.com/vishvananda/netlink"
)

const defaultBrName = "cni0"

type NetConf struct {
	types.NetConf
	BrName string `json:"bridge"`
	IsGW   bool   `json:"isGateway"`
	IPMasq bool   `json:"ipMasq"`
	MTU    int    `json:"mtu"`
}

func init() {
	// this ensures that main runs only on main thread (thread group leader).
	// since namespace ops (unshare, setns) are done for a single thread, we
	// must ensure that the goroutine does not jump from OS thread to thread
	runtime.LockOSThread()
}

func loadNetConf(bytes []byte) (*NetConf, error) {
	n := &NetConf{
		BrName: defaultBrName,
	}
	if err := json.Unmarshal(bytes, n); err != nil {
		return nil, fmt.Errorf("failed to load netconf: %v", err)
	}
	return n, nil
}

func ensureBridgeAddr(br *netlink.Bridge, ipn *net.IPNet) error {
	addrs, err := netlink.AddrList(br, syscall.AF_INET)
	if err != nil && err != syscall.ENOENT {
		return fmt.Errorf("could not get list of IP addresses: %v", err)
	}

	// if there're no addresses on the bridge, it's ok -- we'll add one
	if len(addrs) > 0 {
		ipnStr := ipn.String()
		for _, a := range addrs {
			// string comp is actually easiest for doing IPNet comps
			if a.IPNet.String() == ipnStr {
				return nil
			}
		}
		return fmt.Errorf("%q already has an IP address different from %v", br.Name, ipn.String())
	}

	addr := &netlink.Addr{IPNet: ipn, Label: ""}
	if err := netlink.AddrAdd(br, addr); err != nil {
		return fmt.Errorf("could not add IP address to %q: %v", br.Name, err)
	}
	return nil
}

func bridgeByName(name string) (*netlink.Bridge, error) {
	l, err := netlink.LinkByName(name)
	if err != nil {
		return nil, fmt.Errorf("could not lookup %q: %v", name, err)
	}
	br, ok := l.(*netlink.Bridge)
	if !ok {
		return nil, fmt.Errorf("%q already exists but is not a bridge", name)
	}
	return br, nil
}

func ensureBridge(brName string, mtu int) (*netlink.Bridge, error) {
	br := &netlink.Bridge{
		LinkAttrs: netlink.LinkAttrs{
			Name: brName,
			MTU:  mtu,
		},
	}

	if err := netlink.LinkAdd(br); err != nil {
		if err != syscall.EEXIST {
			return nil, fmt.Errorf("could not add %q: %v", brName, err)
		}

		// it's ok if the device already exists as long as config is similar
		br, err = bridgeByName(brName)
		if err != nil {
			return nil, err
		}
	}

	if err := netlink.LinkSetUp(br); err != nil {
		return nil, err
	}

	return br, nil
}

func setupVeth(netns string, br *netlink.Bridge, ifName string, mtu int) error {
	var hostVethName string

	err := ns.WithNetNSPath(netns, false, func(hostNS *os.File) error {
		// create the veth pair in the container and move host end into host netns
		hostVeth, _, err := ip.SetupVeth(ifName, mtu, hostNS)
		if err != nil {
			return err
		}

		hostVethName = hostVeth.Attrs().Name
		return nil
	})
	if err != nil {
		return err
	}

	// need to lookup hostVeth again as its index has changed during ns move
	hostVeth, err := netlink.LinkByName(hostVethName)
	if err != nil {
		return fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
	}

	// connect host veth end to the bridge
	if err = netlink.LinkSetMaster(hostVeth, br); err != nil {
		return fmt.Errorf("failed to connect %q to bridge %v: %v", hostVethName, br.Attrs().Name, err)
	}

	return nil
}

func calcGatewayIP(ipn *net.IPNet) net.IP {
	nid := ipn.IP.Mask(ipn.Mask)
	return ip.NextIP(nid)
}

func setupBridge(n *NetConf) (*netlink.Bridge, error) {
	// create bridge if necessary
	br, err := ensureBridge(n.BrName, n.MTU)
	if err != nil {
		return nil, fmt.Errorf("failed to create bridge %q: %v", n.BrName, err)
	}

	return br, nil
}

func cmdAdd(args *skel.CmdArgs) error {
	n, err := loadNetConf(args.StdinData)
	if err != nil {
		return err
	}

	br, err := setupBridge(n)
	if err != nil {
		return err
	}

	if err = setupVeth(args.Netns, br, args.IfName, n.MTU); err != nil {
		return err
	}

	// run the IPAM plugin and get back the config to apply
	result, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
	if err != nil {
		return err
	}

	if result.IP4 == nil {
		return errors.New("IPAM plugin returned missing IPv4 config")
	}

	if result.IP4.Gateway == nil && n.IsGW {
		result.IP4.Gateway = calcGatewayIP(&result.IP4.IP)
	}

	err = ns.WithNetNSPath(args.Netns, false, func(hostNS *os.File) error {
		return ipam.ConfigureIface(args.IfName, result)
	})
	if err != nil {
		return err
	}

	if n.IsGW {
		gwn := &net.IPNet{
			IP:   result.IP4.Gateway,
			Mask: result.IP4.IP.Mask,
		}

		if err = ensureBridgeAddr(br, gwn); err != nil {
			return err
		}

		if err := ip.EnableIP4Forward(); err != nil {
			return fmt.Errorf("failed to enable forwarding: %v", err)
		}
	}

	if n.IPMasq {
		chain := utils.FormatChainName(n.Name, args.ContainerID)
		comment := utils.FormatComment(n.Name, args.ContainerID)
		if err = ip.SetupIPMasq(ip.Network(&result.IP4.IP), chain, comment); err != nil {
			return err
		}
	}

	result.DNS = n.DNS
	return result.Print()
}

func cmdDel(args *skel.CmdArgs) error {
	n, err := loadNetConf(args.StdinData)
	if err != nil {
		return err
	}

	var ipn *net.IPNet
	err = ns.WithNetNSPath(args.Netns, false, func(hostNS *os.File) error {
		var err error
		ipn, err = ip.DelLinkByNameAddr(args.IfName, netlink.FAMILY_V4)
		return err
	})
	if err != nil {
		return err
	}

	if n.IPMasq {
		chain := utils.FormatChainName(n.Name, args.ContainerID)
		comment := utils.FormatComment(n.Name, args.ContainerID)
		if err = ip.TeardownIPMasq(ipn, chain, comment); err != nil {
			return err
		}
	}

	return ipam.ExecDel(n.IPAM.Type, args.StdinData)
}

func main() {
	skel.PluginMain(cmdAdd, cmdDel)
}
<|endoftext|>


<commit_before>package sched

import (
	"bytes"
	"log"
	"time"

	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/expr"
)

// Poll dispatches notification checks when needed.
func (s *Schedule) Poll() {
	var timeout time.Duration = time.Hour
	for {
		// Wait for one of these two.
		select {
		case <-time.After(timeout):
		case <-s.nc:
		}
		timeout = s.CheckNotifications()
		s.Save()
	}
}

func (s *Schedule) Notify(st *State, n *conf.Notification) {
	if s.notifications == nil {
		s.notifications = make(map[*conf.Notification][]*State)
	}
	s.notifications[n] = append(s.notifications[n], st)
}

// CheckNotifications processes past notification events. It returns the
// duration until the soonest notification triggers.
func (s *Schedule) CheckNotifications() time.Duration {
	s.Lock()
	defer s.Unlock()
	timeout := time.Hour
	notifications := s.Notifications
	s.Notifications = nil
	for ak, ns := range notifications {
		for name, t := range ns {
			n, present := s.Conf.Notifications[name]
			if !present {
				continue
			}
			remaining := t.Add(n.Timeout).Sub(time.Now())
			if remaining > 0 {
				if remaining < timeout {
					timeout = remaining
				}
				s.AddNotification(ak, n, t)
				continue
			}
			st := s.status[ak]
			if st == nil {
				continue
			}
			s.Notify(st, n)
			if n.Timeout < timeout {
				timeout = n.Timeout
			}
		}
	}
	s.sendNotifications()
	s.notifications = nil
	return timeout
}

func (s *Schedule) sendNotifications() {
	if s.Conf.Quiet {
		return
	}
	for n, states := range s.notifications {
		ustates := make(States)
		for _, st := range states {
			ak := st.AlertKey()
			if st.Last().Status == StUnknown {
				ustates[ak] = st
			} else {
				s.notify(st, n)
			}
			if n.Next != nil {
				s.AddNotification(ak, n, time.Now().UTC())
			}
		}
		for name, group := range ustates.GroupSets() {
			s.unotify(name, group, n)
		}
	}
}

func (s *Schedule) notify(st *State, n *conf.Notification) {
	a := s.Conf.Alerts[st.Alert]
	subject := new(bytes.Buffer)
	if err := s.ExecuteSubject(subject, a, st); err != nil {
		log.Println(err)
		subject = bytes.NewBufferString(err.Error())
	}
	body := new(bytes.Buffer)
	attachments, err := s.ExecuteBody(body, a, st, true)
	if err != nil {
		log.Println(err)
		body = bytes.NewBufferString(err.Error())
	}
	n.Notify(subject.Bytes(), body.Bytes(), s.Conf, string(st.AlertKey()), attachments...)
}

func (s *Schedule) unotify(name string, group expr.AlertKeys, n *conf.Notification) {
	subject := new(bytes.Buffer)
	body := new(bytes.Buffer)
	now := time.Now().UTC()
	s.Group[now] = group
	if t := s.Conf.UnknownTemplate; t != nil {
		data := s.unknownData(now, name, group)
		if t.Body != nil {
			if err := t.Body.Execute(body, &data); err != nil {
				log.Println("unknown template error:", err)
			}
		}
		if t.Subject != nil {
			if err := t.Subject.Execute(subject, &data); err != nil {
				log.Println("unknown template error:", err)
			}
		}
	}
	n.Notify(subject.Bytes(), body.Bytes(), s.Conf, name)
}

func (s *Schedule) AddNotification(ak expr.AlertKey, n *conf.Notification, started time.Time) {
	if s.Notifications == nil {
		s.Notifications = make(map[expr.AlertKey]map[string]time.Time)
	}
	if s.Notifications[ak] == nil {
		s.Notifications[ak] = make(map[string]time.Time)
	}
	s.Notifications[ak][n.Name] = started
}
<commit_msg>Correct notification next timeout calculation<commit_after>package sched

import (
	"bytes"
	"log"
	"time"

	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/expr"
)

// Poll dispatches notification checks when needed.
func (s *Schedule) Poll() {
	for {
		timeout := s.CheckNotifications()
		s.Save()
		// Wait for one of these two.
		select {
		case <-time.After(timeout):
		case <-s.nc:
		}
	}
}

func (s *Schedule) Notify(st *State, n *conf.Notification) {
	if s.notifications == nil {
		s.notifications = make(map[*conf.Notification][]*State)
	}
	s.notifications[n] = append(s.notifications[n], st)
}

// CheckNotifications processes past notification events. It returns the
// duration until the soonest notification triggers.
func (s *Schedule) CheckNotifications() time.Duration {
	s.Lock()
	defer s.Unlock()
	notifications := s.Notifications
	s.Notifications = nil
	for ak, ns := range notifications {
		for name, t := range ns {
			n, present := s.Conf.Notifications[name]
			if !present {
				continue
			}
			remaining := t.Add(n.Timeout).Sub(time.Now())
			if remaining > 0 {
				s.AddNotification(ak, n, t)
				continue
			}
			st := s.status[ak]
			if st == nil {
				continue
			}
			s.Notify(st, n)
		}
	}
	s.sendNotifications()
	s.notifications = nil
	timeout := time.Hour
	now := time.Now()
	for _, ns := range s.Notifications {
		for name, t := range ns {
			n, present := s.Conf.Notifications[name]
			if !present {
				continue
			}
			remaining := t.Add(n.Timeout).Sub(now)
			if remaining < timeout {
				timeout = remaining
			}
		}
	}
	return timeout
}

func (s *Schedule) sendNotifications() {
	if s.Conf.Quiet {
		return
	}
	for n, states := range s.notifications {
		ustates := make(States)
		for _, st := range states {
			ak := st.AlertKey()
			if st.Last().Status == StUnknown {
				ustates[ak] = st
			} else {
				s.notify(st, n)
			}
			if n.Next != nil {
				s.AddNotification(ak, n, time.Now().UTC())
			}
		}
		for name, group := range ustates.GroupSets() {
			s.unotify(name, group, n)
		}
	}
}

func (s *Schedule) notify(st *State, n *conf.Notification) {
	a := s.Conf.Alerts[st.Alert]
	subject := new(bytes.Buffer)
	if err := s.ExecuteSubject(subject, a, st); err != nil {
		log.Println(err)
		subject = bytes.NewBufferString(err.Error())
	}
	body := new(bytes.Buffer)
	attachments, err := s.ExecuteBody(body, a, st, true)
	if err != nil {
		log.Println(err)
		body = bytes.NewBufferString(err.Error())
	}
	n.Notify(subject.Bytes(), body.Bytes(), s.Conf, string(st.AlertKey()), attachments...)
}

func (s *Schedule) unotify(name string, group expr.AlertKeys, n *conf.Notification) {
	subject := new(bytes.Buffer)
	body := new(bytes.Buffer)
	now := time.Now().UTC()
	s.Group[now] = group
	if t := s.Conf.UnknownTemplate; t != nil {
		data := s.unknownData(now, name, group)
		if t.Body != nil {
			if err := t.Body.Execute(body, &data); err != nil {
				log.Println("unknown template error:", err)
			}
		}
		if t.Subject != nil {
			if err := t.Subject.Execute(subject, &data); err != nil {
				log.Println("unknown template error:", err)
			}
		}
	}
	n.Notify(subject.Bytes(), body.Bytes(), s.Conf, name)
}

func (s *Schedule) AddNotification(ak expr.AlertKey, n *conf.Notification, started time.Time) {
	if s.Notifications == nil {
		s.Notifications = make(map[expr.AlertKey]map[string]time.Time)
	}
	if s.Notifications[ak] == nil {
		s.Notifications[ak] = make(map[string]time.Time)
	}
	s.Notifications[ak][n.Name] = started
}
<|endoftext|>


<commit_before>package gostore

/**
Store is deprecated
*/
type Store interface {
	Get(key []byte, resource string) ([][]byte, error)
	PrefixGet(prefix []byte, resource string) ([][]byte, error) //Gets an item with a prefix
	Save(key []byte, obj []byte, resource string) error
	Delete(key []byte, resource string) error
	DeleteAll(resource string) error
	GetAll(count int, skip int, resource string) ([][][]byte, error)
	GetAllAfter(key []byte, count int, skip int, resource string) ([][][]byte, error)  //Get all items after a key
	GetAllBefore(key []byte, count int, skip int, resource string) ([][][]byte, error) //Get all items before a key
	Filter(prefix []byte, count int, skip int, resource string) ([][][]byte, error)

	//Streaming api
	StreamFilter(key []byte, count int, resource string) chan []byte
	StreamAll(count int, resource string) chan [][]byte //Stream all entries through a channel

	//Misc api
	Stats(bucket string) (map[string]interface{}, error)
	GetStoreObject() interface{}
}

type ObjectStoreOptions interface {
	GetIndexes() map[string][]string
}

type DefaultObjectStoreOptions struct {
	Index map[string][]string
}

func (d DefaultObjectStoreOptions) GetIndexes() map[string][]string {
	return d.Index
}

//ObjectStore represents all api common to all database implementations
type ObjectStore interface {
	CreateDatabase() error
	CreateTable(table string, sample interface{}) error

	GetStore() interface{}
	Stats(store string) (map[string]interface{}, error)

	All(count int, skip int, store string) (ObjectRows, error)
	AllCursor(store string) (ObjectRows, error)
	AllWithinRange(filter map[string]interface{}, count int, skip int, store string, opts ObjectStoreOptions) (ObjectRows, error)

	Since(id string, count int, skip int, store string) (ObjectRows, error)  //Get all recent items from a key
	Before(id string, count int, skip int, store string) (ObjectRows, error) //Get all existing items before a key

	FilterSince(id string, filter map[string]interface{}, count int, skip int, store string, opts ObjectStoreOptions) (ObjectRows, error)  //Get all recent items from a key
	FilterBefore(id string, filter map[string]interface{}, count int, skip int, store string, opts ObjectStoreOptions) (ObjectRows, error) //Get all existing items before a key
	FilterBeforeCount(id string, filter map[string]interface{}, count int, skip int, store string, opts ObjectStoreOptions) (int64, error) //Get all existing items before a key

	Get(key string, store string, dst interface{}) error
	Save(store string, src interface{}) (string, error)
	SaveAll(store string, src ...interface{}) (keys []string, err error)
	Update(key string, store string, src interface{}) error
	Replace(key string, store string, src interface{}) error
	Delete(key string, store string) error

	FilterUpdate(filter map[string]interface{}, src interface{}, store string, opts ObjectStoreOptions) error
	FilterReplace(filter map[string]interface{}, src interface{}, store string, opts ObjectStoreOptions) error
	FilterGet(filter map[string]interface{}, store string, dst interface{}, opts ObjectStoreOptions) error
	FilterGetAll(filter map[string]interface{}, count int, skip int, store string, opts ObjectStoreOptions) (ObjectRows, error)
	FilterDelete(filter map[string]interface{}, store string, opts ObjectStoreOptions) error
	FilterCount(filter map[string]interface{}, store string, opts ObjectStoreOptions) (int64, error)

	GetByField(name, val, store string, dst interface{}) error
	GetByFieldsByField(name, val, store string, fields []string, dst interface{}) (err error)

	BatchDelete(ids []interface{}, store string, opts ObjectStoreOptions) (err error)
	BatchUpdate(id []interface{}, data []interface{}, store string, opts ObjectStoreOptions) error
	BatchFilterDelete(filter []map[string]interface{}, store string, opts ObjectStoreOptions) error

	Close()
}

type ObjectRows interface {
	Next(interface{}) (bool, error)
	NextRaw() ([]byte, bool)
	Close()
	LastError() error
}

type StoreOptions map[string]interface{}

type StoreObj interface {
	SetKey(key string)
	GetKey() string
}

type StoreObjs []StoreObj

type TableConfig struct {
	NestedBucketFields map[string]string //defines fields to be used to extract nested buckets for data
}
<commit_msg>add BatchInsert interface fn, this will break gostore, till further update<commit_after>package gostore

import "github.com/osiloke/gostore"

/**
Store is deprecated
*/
type Store interface {
	Get(key []byte, resource string) ([][]byte, error)
	PrefixGet(prefix []byte, resource string) ([][]byte, error) //Gets an item with a prefix
	Save(key []byte, obj []byte, resource string) error
	Delete(key []byte, resource string) error
	DeleteAll(resource string) error
	GetAll(count int, skip int, resource string) ([][][]byte, error)
	GetAllAfter(key []byte, count int, skip int, resource string) ([][][]byte, error)  //Get all items after a key
	GetAllBefore(key []byte, count int, skip int, resource string) ([][][]byte, error) //Get all items before a key
	Filter(prefix []byte, count int, skip int, resource string) ([][][]byte, error)

	//Streaming api
	StreamFilter(key []byte, count int, resource string) chan []byte
	StreamAll(count int, resource string) chan [][]byte //Stream all entries through a channel

	//Misc api
	Stats(bucket string) (map[string]interface{}, error)
	GetStoreObject() interface{}
}

type ObjectStoreOptions interface {
	GetIndexes() map[string][]string
}

type DefaultObjectStoreOptions struct {
	Index map[string][]string
}

func (d DefaultObjectStoreOptions) GetIndexes() map[string][]string {
	return d.Index
}

//ObjectStore represents all api common to all database implementations
type ObjectStore interface {
	CreateDatabase() error
	CreateTable(table string, sample interface{}) error

	GetStore() interface{}
	Stats(store string) (map[string]interface{}, error)

	All(count int, skip int, store string) (ObjectRows, error)
	AllCursor(store string) (ObjectRows, error)
	AllWithinRange(filter map[string]interface{}, count int, skip int, store string, opts ObjectStoreOptions) (ObjectRows, error)

	Since(id string, count int, skip int, store string) (ObjectRows, error)  //Get all recent items from a key
	Before(id string, count int, skip int, store string) (ObjectRows, error) //Get all existing items before a key

	FilterSince(id string, filter map[string]interface{}, count int, skip int, store string, opts ObjectStoreOptions) (ObjectRows, error)  //Get all recent items from a key
	FilterBefore(id string, filter map[string]interface{}, count int, skip int, store string, opts ObjectStoreOptions) (ObjectRows, error) //Get all existing items before a key
	FilterBeforeCount(id string, filter map[string]interface{}, count int, skip int, store string, opts ObjectStoreOptions) (int64, error) //Get all existing items before a key

	Get(key string, store string, dst interface{}) error
	Save(store string, src interface{}) (string, error)
	SaveAll(store string, src ...interface{}) (keys []string, err error)
	Update(key string, store string, src interface{}) error
	Replace(key string, store string, src interface{}) error
	Delete(key string, store string) error

	FilterUpdate(filter map[string]interface{}, src interface{}, store string, opts ObjectStoreOptions) error
	FilterReplace(filter map[string]interface{}, src interface{}, store string, opts ObjectStoreOptions) error
	FilterGet(filter map[string]interface{}, store string, dst interface{}, opts ObjectStoreOptions) error
	FilterGetAll(filter map[string]interface{}, count int, skip int, store string, opts ObjectStoreOptions) (ObjectRows, error)
	FilterDelete(filter map[string]interface{}, store string, opts ObjectStoreOptions) error
	FilterCount(filter map[string]interface{}, store string, opts ObjectStoreOptions) (int64, error)

	GetByField(name, val, store string, dst interface{}) error
	GetByFieldsByField(name, val, store string, fields []string, dst interface{}) (err error)

	BatchDelete(ids []interface{}, store string, opts ObjectStoreOptions) (err error)
	BatchUpdate(id []interface{}, data []interface{}, store string, opts ObjectStoreOptions) error
	BatchFilterDelete(filter []map[string]interface{}, store string, opts ObjectStoreOptions) error
	BatchInsert(data []interface{}, store string, opts gostore.ObjectStoreOptions) (keys []string, err error)

	Close()
}

type ObjectRows interface {
	Next(interface{}) (bool, error)
	NextRaw() ([]byte, bool)
	Close()
	LastError() error
}

type StoreOptions map[string]interface{}

type StoreObj interface {
	SetKey(key string)
	GetKey() string
}

type StoreObjs []StoreObj

type TableConfig struct {
	NestedBucketFields map[string]string //defines fields to be used to extract nested buckets for data
}
<|endoftext|>