<commit_before>package stasher

import (
  "context"
  "time"
  "net/http"
  "fmt"
  "google.golang.org/api/blogger/v3"
  "google.golang.org/api/googleapi"
  "golang.org/x/oauth2/google"
  "gopkg.in/mgo.v2"
  "gopkg.in/mgo.v2/bson"
)

const (
  DefaultBlogPageCollection = "blog_page"
  DefaultBlogPostCollection = "blog_post"
  DefaultBlogCollection = "blog"
)

type BloggerStasher interface {
  BlogPostStasher
  BlogPageStasher
  BlogStasher
}

type BlogPostStasher interface {
  GetPostListEtag(ctx context.Context, blogId string) (etag *string)
  HasPost(ctx context.Context, pageId string, etag string) bool
  StashPost(context.Context, *blogger.Post)
  StashPostEtags(ctx context.Context, blogId string, listEtag string, postEtags map[string]string, updated *time.Time)
}

type BlogPageStasher interface {
  GetPageListEtag(ctx context.Context, blogId string) (etag *string)
  HasPage(ctx context.Context, pageId string, etag string) bool
  StashPage(context.Context, *blogger.Page)
  StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, updated *time.Time)
}

type BlogStasher interface {
  StashBlog(context.Context, *blogger.Blog)
}

type MgoBlogStasher struct {
  BlogPageCollection *mgo.Collection
  BlogPostCollection *mgo.Collection
  BlogCollection *mgo.Collection
}

type MgoBlogPost struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPost *blogger.Post
  Updated *time.Time
}

type MgoBlogPage struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPage *blogger.Page
  Updated *time.Time
}

type MgoBlog struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  Blog *blogger.Blog `bson:",omitempty"`
  Updated *time.Time `bson:",omitempty"`
  PostListEtag *string `bson:",omitempty"`
  PostListUpdated *time.Time `bson:",omitempty"`
  PageListEtag *string `bson:",omitempty"`
  PageListUpdated *time.Time `bson:",omitempty"`
}

func DefaultMgoStasher(db *mgo.Database) (m *MgoBlogStasher) {
  m = new(MgoBlogStasher)
  m.BlogPageCollection = db.C(DefaultBlogPageCollection)
  m.BlogPostCollection = db.C(DefaultBlogPostCollection)
  m.BlogCollection = db.C(DefaultBlogCollection)
  return
}

type blogPostGetter interface {
  blogPostEtags(blogId string, oldListEtag *string) (newListEtag string, newPostEtags map[string]string)
  blogPost(blogId string, postId string) *blogger.Post
}

type blogPageGetter interface {
  blogPageEtags(blogId string, oldListEtag *string) (newListEtag string, newPageEtags map[string]string)
  blogPage(blogId string, pageId string) *blogger.Page
}
type blogGetter interface {
  blog(blogId string) *blogger.Blog
}

type blogService blogger.Service;

func(b *blogService) blogPostEtags(blogId string, oldListEtag *string) (newListEtag string, newPostEtags map[string]string) {
  s := (*blogger.Service)(b)
  postListCall := s.Posts.List(blogId).FetchBodies(false)
  if oldListEtag != nil {
    postListCall = postListCall.IfNoneMatch(*oldListEtag)
  }
  newPostEtags = make(map[string]string)
  err := postListCall.Pages(nil, func(pList *blogger.PostList) error {
    newListEtag = pList.Etag
    for _, p := range pList.Items {
      newPostEtags[p.Id] = p.Etag
    }
    return nil
  })
  if err != nil && ! googleapi.IsNotModified(err) {
    panic(err)
  }
  return
}

func(b *blogService) blogPageEtags(blogId string, oldListEtag *string) (newListEtag string, newPageEtags map[string]string) {
  s := (*blogger.Service)(b)
  pageListCall := s.Pages.List(blogId).FetchBodies(false)
  if oldListEtag != nil {
    pageListCall = pageListCall.IfNoneMatch(*oldListEtag)
  }
  newPageEtags = make(map[string]string)
  err := pageListCall.Pages(nil, func(pList *blogger.PageList) error {
    newListEtag = pList.Etag
    for _, p := range pList.Items {
      newPageEtags[p.Id] = p.Etag
    }
    return nil
  })
  if err != nil && ! googleapi.IsNotModified(err) {
    panic(err)
  }
  return
}

func(b *blogService) blogPost(blogId string, postId string) *blogger.Post {
  s := (*blogger.Service)(b)
  post, err := s.Posts.Get(blogId, postId).FetchImages(true).Do()
  if err != nil { panic(err) }
  return post
}

func(b *blogService) blogPage(blogId string, pageId string) *blogger.Page {
  s := (*blogger.Service)(b)
  page, err := s.Pages.Get(blogId, pageId).Do()
  if err != nil { panic(err) }
  return page
}

func(b *blogService) blog(blogId string) *blogger.Blog {
  s := (*blogger.Service)(b)
  blog, err := s.Blogs.Get(blogId).Do()
  if err != nil { panic(err) }
  return blog
}

func SyncBloggerWithDefaultClient(ctx context.Context, blogId string, stasher BloggerStasher) error{
  client, err := google.DefaultClient(ctx, blogger.BloggerReadonlyScope)
  if err != nil { return err }
  return syncBloggerWithClient(ctx, blogId, stasher, client)
}

func SyncBloggerWithCredentials(ctx context.Context, blogId string, stasher BloggerStasher, credentials []byte) error{
  jwtConf, err := google.JWTConfigFromJSON(credentials, blogger.BloggerReadonlyScope)
  if err != nil { return err }
  client := jwtConf.Client(ctx)
  return syncBloggerWithClient(ctx, blogId, stasher, client)
}

func syncBloggerWithClient(ctx context.Context, blogId string, stasher BloggerStasher, client *http.Client) (err error) {

  defer func() {
    if r := recover(); r != nil {
      var ok bool
      err, ok = r.(error)
      if !ok {
        err = fmt.Errorf("pkg: %v", r)
      }
    }
  }()

  var b *blogService
  if bService, err := blogger.New(client); err != nil {
    return err
  } else {
    b = (*blogService)(bService)
  }
  syncBlog(ctx, blogId, stasher, b)
  syncBlogPosts(ctx, blogId, stasher, b)
  syncBlogPages(ctx, blogId, stasher, b)
  return nil
}

func syncBlog(ctx context.Context, blogId string, stasher BlogStasher, getter blogGetter) {
  newBlog := getter.blog(blogId)
  stasher.StashBlog(ctx, newBlog)
}

func syncBlogPosts(ctx context.Context, blogId string, stasher BlogPostStasher, getter blogPostGetter) {
  oldListEtag := stasher.GetPostListEtag(ctx, blogId)
  newListEtag, newPostEtags := getter.blogPostEtags(blogId, oldListEtag)
  if newListEtag == "" { return }
  newUpdated := time.Now()
  for id, etag := range newPostEtags {
    if stasher.HasPost(ctx, id, etag) {continue}
    post := getter.blogPost(blogId, id)
    stasher.StashPost(ctx, post)
  }
  stasher.StashPostEtags(ctx, blogId, newListEtag, newPostEtags, &newUpdated)
}

func syncBlogPages(ctx context.Context, blogId string, stasher BlogPageStasher, getter blogPageGetter) {
  oldListEtag := stasher.GetPageListEtag(ctx, blogId)
  newListEtag, newPageEtags := getter.blogPageEtags(blogId, oldListEtag)
  if newListEtag == "" { return }
  newUpdated := time.Now()
  for id, etag := range newPageEtags {
    if stasher.HasPage(ctx, id, etag) {continue}
    page := getter.blogPage(blogId, id)
    stasher.StashPage(ctx, page)
  }
  stasher.StashPageEtags(ctx, blogId, newListEtag, newPageEtags, &newUpdated)
}

func(m *MgoBlogStasher) GetPageListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(MgoBlog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"pageListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PageListEtag
}
func(m *MgoBlogStasher) HasPage(ctx context.Context, pageId string, etag string) bool {
  n, err := m.BlogPageCollection.Find(bson.M{"blogPage.id": pageId, "blogPage.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}
func mgoWrapBlogPage(page *blogger.Page) *MgoBlogPage {
  dbPage := &MgoBlogPage{
    BlogPage: page,
  }
  pageUpdated, err := time.Parse(time.RFC3339, page.Updated)
  if err != nil { panic(err) }
  dbPage.Updated = &pageUpdated
  return dbPage
}
func(m *MgoBlogStasher) StashPage(ctx context.Context, page *blogger.Page) {
  dbPage := mgoWrapBlogPage(page)
  _, err := m.BlogPageCollection.Upsert(bson.M{"blogPage.id": page.Id, "updated": bson.M{"$lt": dbPage.Updated}}, &dbPage);
  if err != nil { panic(err) }
}
func(m *MgoBlogStasher) StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPageCollection.Find(nil).Select(bson.M{"dbPage.id": 1, "dbPage.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPage := new(MgoBlogPage)
  for iter.Next(dbPage) {
    if pageEtags[dbPage.BlogPage.Id] != dbPage.BlogPage.Etag {
      if err := m.BlogPageCollection.RemoveId(dbPage.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "pageListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &MgoBlog{PageListEtag: &listEtag, PageListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *MgoBlogStasher) GetPostListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(MgoBlog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"postListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PostListEtag
}
func(m *MgoBlogStasher) HasPost(ctx context.Context, postId string, etag string) bool {
  n, err := m.BlogPostCollection.Find(bson.M{"blogPost.id": postId, "blogPost.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}

func mgoWrapBlogPost(post *blogger.Post) *MgoBlogPost {
  dbPost := &MgoBlogPost{
    BlogPost: post,
  }
  postUpdated, err := time.Parse(time.RFC3339, post.Updated)
  if err != nil { panic(err) }
  dbPost.Updated = &postUpdated
  return dbPost
}
func(m *MgoBlogStasher) StashPost(ctx context.Context, post *blogger.Post) {
  dbPost := mgoWrapBlogPost(post)
  _, err := m.BlogPostCollection.Upsert(bson.M{"blogPost.id": post.Id, "updated": bson.M{"$lt": dbPost.Updated}}, dbPost);
  if err != nil { panic(err) }
}
func(m *MgoBlogStasher) StashPostEtags(ctx context.Context, blogId string, listEtag string, postEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPostCollection.Find(nil).Select(bson.M{"dbPost.id": 1, "dbPost.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPost := new(MgoBlogPost)
  for iter.Next(dbPost) {
    if postEtags[dbPost.BlogPost.Id] != dbPost.BlogPost.Etag {
      if err := m.BlogPostCollection.RemoveId(dbPost.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "postListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &MgoBlog{PostListEtag: &listEtag, PostListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *MgoBlogStasher) StashBlog(ctx context.Context, blog *blogger.Blog) {
  dbBlog := MgoBlog{
    Blog: blog,
  }
  var err error
  *dbBlog.Updated, err = time.Parse(time.RFC3339, blog.Updated)
  if err != nil { panic(err) }
  _, err = m.BlogCollection.Upsert(bson.M{"blog.id": blog.Id, "updated": bson.M{"$lt": dbBlog.Updated}}, bson.M{"$set": &dbBlog})
  if (err != nil) { panic(err) }
}
<commit_msg>Separated out client from func<commit_after>package stasher

import (
  "context"
  "time"
  "net/http"
  "fmt"
  "google.golang.org/api/blogger/v3"
  "google.golang.org/api/googleapi"
  "golang.org/x/oauth2/google"
  "gopkg.in/mgo.v2"
  "gopkg.in/mgo.v2/bson"
)

const (
  DefaultBlogPageCollection = "blog_page"
  DefaultBlogPostCollection = "blog_post"
  DefaultBlogCollection = "blog"
)

type BloggerStasher interface {
  BlogPostStasher
  BlogPageStasher
  BlogStasher
}

type BlogPostStasher interface {
  GetPostListEtag(ctx context.Context, blogId string) (etag *string)
  HasPost(ctx context.Context, pageId string, etag string) bool
  StashPost(context.Context, *blogger.Post)
  StashPostEtags(ctx context.Context, blogId string, listEtag string, postEtags map[string]string, updated *time.Time)
}

type BlogPageStasher interface {
  GetPageListEtag(ctx context.Context, blogId string) (etag *string)
  HasPage(ctx context.Context, pageId string, etag string) bool
  StashPage(context.Context, *blogger.Page)
  StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, updated *time.Time)
}

type BlogStasher interface {
  StashBlog(context.Context, *blogger.Blog)
}

type MgoBlogStasher struct {
  BlogPageCollection *mgo.Collection
  BlogPostCollection *mgo.Collection
  BlogCollection *mgo.Collection
}

type MgoBlogPost struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPost *blogger.Post
  Updated *time.Time
}

type MgoBlogPage struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPage *blogger.Page
  Updated *time.Time
}

type MgoBlog struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  Blog *blogger.Blog `bson:",omitempty"`
  Updated *time.Time `bson:",omitempty"`
  PostListEtag *string `bson:",omitempty"`
  PostListUpdated *time.Time `bson:",omitempty"`
  PageListEtag *string `bson:",omitempty"`
  PageListUpdated *time.Time `bson:",omitempty"`
}

func DefaultMgoStasher(db *mgo.Database) (m *MgoBlogStasher) {
  m = new(MgoBlogStasher)
  m.BlogPageCollection = db.C(DefaultBlogPageCollection)
  m.BlogPostCollection = db.C(DefaultBlogPostCollection)
  m.BlogCollection = db.C(DefaultBlogCollection)
  return
}

type blogPostGetter interface {
  blogPostEtags(blogId string, oldListEtag *string) (newListEtag string, newPostEtags map[string]string)
  blogPost(blogId string, postId string) *blogger.Post
}

type blogPageGetter interface {
  blogPageEtags(blogId string, oldListEtag *string) (newListEtag string, newPageEtags map[string]string)
  blogPage(blogId string, pageId string) *blogger.Page
}
type blogGetter interface {
  blog(blogId string) *blogger.Blog
}

type blogService blogger.Service;

func(b *blogService) blogPostEtags(blogId string, oldListEtag *string) (newListEtag string, newPostEtags map[string]string) {
  s := (*blogger.Service)(b)
  postListCall := s.Posts.List(blogId).FetchBodies(false)
  if oldListEtag != nil {
    postListCall = postListCall.IfNoneMatch(*oldListEtag)
  }
  newPostEtags = make(map[string]string)
  err := postListCall.Pages(nil, func(pList *blogger.PostList) error {
    newListEtag = pList.Etag
    for _, p := range pList.Items {
      newPostEtags[p.Id] = p.Etag
    }
    return nil
  })
  if err != nil && ! googleapi.IsNotModified(err) {
    panic(err)
  }
  return
}

func(b *blogService) blogPageEtags(blogId string, oldListEtag *string) (newListEtag string, newPageEtags map[string]string) {
  s := (*blogger.Service)(b)
  pageListCall := s.Pages.List(blogId).FetchBodies(false)
  if oldListEtag != nil {
    pageListCall = pageListCall.IfNoneMatch(*oldListEtag)
  }
  newPageEtags = make(map[string]string)
  err := pageListCall.Pages(nil, func(pList *blogger.PageList) error {
    newListEtag = pList.Etag
    for _, p := range pList.Items {
      newPageEtags[p.Id] = p.Etag
    }
    return nil
  })
  if err != nil && ! googleapi.IsNotModified(err) {
    panic(err)
  }
  return
}

func(b *blogService) blogPost(blogId string, postId string) *blogger.Post {
  s := (*blogger.Service)(b)
  post, err := s.Posts.Get(blogId, postId).FetchImages(true).Do()
  if err != nil { panic(err) }
  return post
}

func(b *blogService) blogPage(blogId string, pageId string) *blogger.Page {
  s := (*blogger.Service)(b)
  page, err := s.Pages.Get(blogId, pageId).Do()
  if err != nil { panic(err) }
  return page
}

func(b *blogService) blog(blogId string) *blogger.Blog {
  s := (*blogger.Service)(b)
  blog, err := s.Blogs.Get(blogId).Do()
  if err != nil { panic(err) }
  return blog
}

func DefaultSyncBloggerClient (ctx context.Context) (*http.Client, error) {
  return google.DefaultClient(ctx, blogger.BloggerReadonlyScope)
}

func SyncBloggerClient(ctx context.Context, credentials []byte) (*http.Client, error) {
  jwtConf, err := google.JWTConfigFromJSON(credentials, blogger.BloggerReadonlyScope)
  if err != nil { return nil, err }
  return jwtConf.Client(ctx), nil
}

func SyncBlogger(ctx context.Context, blogId string, stasher BloggerStasher, client *http.Client) (err error) {

  defer func() {
    if r := recover(); r != nil {
      var ok bool
      err, ok = r.(error)
      if !ok {
        err = fmt.Errorf("pkg: %v", r)
      }
    }
  }()

  var b *blogService
  if bService, err := blogger.New(client); err != nil {
    return err
  } else {
    b = (*blogService)(bService)
  }
  syncBlog(ctx, blogId, stasher, b)
  syncBlogPosts(ctx, blogId, stasher, b)
  syncBlogPages(ctx, blogId, stasher, b)
  return nil
}

func syncBlog(ctx context.Context, blogId string, stasher BlogStasher, getter blogGetter) {
  newBlog := getter.blog(blogId)
  stasher.StashBlog(ctx, newBlog)
}

func syncBlogPosts(ctx context.Context, blogId string, stasher BlogPostStasher, getter blogPostGetter) {
  oldListEtag := stasher.GetPostListEtag(ctx, blogId)
  newListEtag, newPostEtags := getter.blogPostEtags(blogId, oldListEtag)
  if newListEtag == "" { return }
  newUpdated := time.Now()
  for id, etag := range newPostEtags {
    if stasher.HasPost(ctx, id, etag) {continue}
    post := getter.blogPost(blogId, id)
    stasher.StashPost(ctx, post)
  }
  stasher.StashPostEtags(ctx, blogId, newListEtag, newPostEtags, &newUpdated)
}

func syncBlogPages(ctx context.Context, blogId string, stasher BlogPageStasher, getter blogPageGetter) {
  oldListEtag := stasher.GetPageListEtag(ctx, blogId)
  newListEtag, newPageEtags := getter.blogPageEtags(blogId, oldListEtag)
  if newListEtag == "" { return }
  newUpdated := time.Now()
  for id, etag := range newPageEtags {
    if stasher.HasPage(ctx, id, etag) {continue}
    page := getter.blogPage(blogId, id)
    stasher.StashPage(ctx, page)
  }
  stasher.StashPageEtags(ctx, blogId, newListEtag, newPageEtags, &newUpdated)
}

func(m *MgoBlogStasher) GetPageListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(MgoBlog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"pageListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PageListEtag
}
func(m *MgoBlogStasher) HasPage(ctx context.Context, pageId string, etag string) bool {
  n, err := m.BlogPageCollection.Find(bson.M{"blogPage.id": pageId, "blogPage.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}
func mgoWrapBlogPage(page *blogger.Page) *MgoBlogPage {
  dbPage := &MgoBlogPage{
    BlogPage: page,
  }
  pageUpdated, err := time.Parse(time.RFC3339, page.Updated)
  if err != nil { panic(err) }
  dbPage.Updated = &pageUpdated
  return dbPage
}
func(m *MgoBlogStasher) StashPage(ctx context.Context, page *blogger.Page) {
  dbPage := mgoWrapBlogPage(page)
  _, err := m.BlogPageCollection.Upsert(bson.M{"blogPage.id": page.Id, "updated": bson.M{"$lt": dbPage.Updated}}, &dbPage);
  if err != nil { panic(err) }
}
func(m *MgoBlogStasher) StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPageCollection.Find(nil).Select(bson.M{"dbPage.id": 1, "dbPage.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPage := new(MgoBlogPage)
  for iter.Next(dbPage) {
    if pageEtags[dbPage.BlogPage.Id] != dbPage.BlogPage.Etag {
      if err := m.BlogPageCollection.RemoveId(dbPage.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "pageListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &MgoBlog{PageListEtag: &listEtag, PageListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *MgoBlogStasher) GetPostListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(MgoBlog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"postListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PostListEtag
}
func(m *MgoBlogStasher) HasPost(ctx context.Context, postId string, etag string) bool {
  n, err := m.BlogPostCollection.Find(bson.M{"blogPost.id": postId, "blogPost.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}

func mgoWrapBlogPost(post *blogger.Post) *MgoBlogPost {
  dbPost := &MgoBlogPost{
    BlogPost: post,
  }
  postUpdated, err := time.Parse(time.RFC3339, post.Updated)
  if err != nil { panic(err) }
  dbPost.Updated = &postUpdated
  return dbPost
}
func(m *MgoBlogStasher) StashPost(ctx context.Context, post *blogger.Post) {
  dbPost := mgoWrapBlogPost(post)
  _, err := m.BlogPostCollection.Upsert(bson.M{"blogPost.id": post.Id, "updated": bson.M{"$lt": dbPost.Updated}}, dbPost);
  if err != nil { panic(err) }
}
func(m *MgoBlogStasher) StashPostEtags(ctx context.Context, blogId string, listEtag string, postEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPostCollection.Find(nil).Select(bson.M{"dbPost.id": 1, "dbPost.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPost := new(MgoBlogPost)
  for iter.Next(dbPost) {
    if postEtags[dbPost.BlogPost.Id] != dbPost.BlogPost.Etag {
      if err := m.BlogPostCollection.RemoveId(dbPost.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "postListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &MgoBlog{PostListEtag: &listEtag, PostListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *MgoBlogStasher) StashBlog(ctx context.Context, blog *blogger.Blog) {
  dbBlog := MgoBlog{
    Blog: blog,
  }
  var err error
  *dbBlog.Updated, err = time.Parse(time.RFC3339, blog.Updated)
  if err != nil { panic(err) }
  _, err = m.BlogCollection.Upsert(bson.M{"blog.id": blog.Id, "updated": bson.M{"$lt": dbBlog.Updated}}, bson.M{"$set": &dbBlog})
  if (err != nil) { panic(err) }
}
<|endoftext|>


<commit_before>package main

import (
	"sync"

	"github.com/gdamore/tcell"
	"github.com/gdamore/tcell/views"
	"github.com/viktomas/godu/core"
)

func InteractiveTree(s tcell.Screen, states chan core.State, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		state, more := <-states
		if !more {
			break
		}
		printOptions(state, s)
	}
}

func printOptions(state core.State, s tcell.Screen) {
	s.Clear()
	inner := views.NewBoxLayout(views.Horizontal)
	var back views.Widget
	var forth views.Widget

	middle := views.NewCellView()

	middle.SetModel(NewVisualState(state))
	backState, err := core.GoBack{}.Execute(state)
	if err == nil {
		backCell := views.NewCellView()
		backCell.SetModel(NewVisualState(backState))
		back = backCell
	} else {
		back = views.NewText()
	}
	forthState, err := core.Enter{}.Execute(state)
	if err == nil {
		forthCell := views.NewCellView()
		forthCell.SetModel(NewVisualState(forthState))
		forth = forthCell
	} else {
		forth = views.NewText()
	}

	inner.AddWidget(back, 0.33)
	inner.AddWidget(middle, 0.33)
	inner.AddWidget(forth, 0.33)
	inner.SetView(s)
	inner.Draw()
	s.Show()
}
<commit_msg>Workaround for PR140 in gdamore/tcell<commit_after>package main

import (
	"sync"

	"github.com/gdamore/tcell"
	"github.com/gdamore/tcell/views"
	"github.com/viktomas/godu/core"
)

func InteractiveTree(s tcell.Screen, states chan core.State, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		state, more := <-states
		if !more {
			break
		}
		printOptions(state, s)
	}
}

func printOptions(state core.State, s tcell.Screen) {
	s.Clear()
	inner := views.NewBoxLayout(views.Horizontal)
	var back views.Widget
	var forth views.Widget

	middle := views.NewCellView()

	middle.SetModel(NewVisualState(state))
	backState, err := core.GoBack{}.Execute(state)
	if err == nil {
		backCell := views.NewCellView()
		backCell.SetModel(NewVisualState(backState))
		back = backCell
	} else {
		back = views.NewText()
	}
	forthState, err := core.Enter{}.Execute(state)
	if err == nil {
		forthCell := views.NewCellView()
		forthCell.SetModel(NewVisualState(forthState))
		forth = forthCell
	} else {
		forth = views.NewText()
	}

	inner.SetView(s)
	inner.AddWidget(back, 0.33)
	inner.AddWidget(middle, 0.33)
	inner.AddWidget(forth, 0.33)
	back.Resize()
	middle.Resize()
	forth.Resize()
	inner.Draw()
	s.Show()
}
<|endoftext|>


<commit_before>package similar

import "github.com/jaeyeom/math/vector"

// wordVector generates word frequency vectors.
type wordVector map[string]int

// strings returns a vector from words.
func (w wordVector) strings(words []string) vector.Vector {
	v := vector.New(len(w))
	for _, word := range words {
		i := len(v)
		if j, ok := w[word]; ok {
			i = j
		} else {
			w[word] = i
		}
		if i >= len(v) {
			v = append(v, make([]int, i+1-len(v))...)
		}
		v[i]++
	}
	return v
}
<commit_msg>Use vector.New rather than make([]int).<commit_after>package similar

import "github.com/jaeyeom/math/vector"

// wordVector generates word frequency vectors.
type wordVector map[string]int

// strings returns a vector from words.
func (w wordVector) strings(words []string) vector.Vector {
	v := vector.New(len(w))
	for _, word := range words {
		i := len(v)
		if j, ok := w[word]; ok {
			i = j
		} else {
			w[word] = i
		}
		if i >= len(v) {
			v = append(v, vector.New(i+1-len(v))...)
		}
		v[i]++
	}
	return v
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/proto"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestDev(t *testing.T) {
	tests := []struct {
		description string
		trigger     string
	}{
		{
			description: "dev with polling trigger",
			trigger:     "polling",
		},
		{
			description: "dev with notify trigger",
			trigger:     "notify",
		},
	}
	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			if testing.Short() {
				t.Skip("skipping integration test")
			}
			if ShouldRunGCPOnlyTests() {
				t.Skip("skipping test that is not gcp only")
			}

			Run(t, "testdata/dev", "sh", "-c", "echo foo > foo")
			defer Run(t, "testdata/dev", "rm", "foo")

			// Run skaffold build first to fail quickly on a build failure
			skaffold.Build().InDir("testdata/dev").RunOrFail(t)

			ns, client, deleteNs := SetupNamespace(t)
			defer deleteNs()

			stop := skaffold.Dev("--trigger", test.trigger).InDir("testdata/dev").InNs(ns.Name).RunBackground(t)
			defer stop()

			dep := client.GetDeployment("test-dev")

			// Make a change to foo so that dev is forced to delete the Deployment and redeploy
			Run(t, "testdata/dev", "sh", "-c", "echo bar > foo")

			// Make sure the old Deployment and the new Deployment are different
			err := wait.PollImmediate(time.Millisecond*500, 10*time.Minute, func() (bool, error) {
				newDep := client.GetDeployment("test-dev")
				return dep.GetGeneration() != newDep.GetGeneration(), nil
			})
			testutil.CheckError(t, false, err)
		})
	}
}

func TestDevAPITriggers(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test")
	}
	if ShouldRunGCPOnlyTests() {
		t.Skip("skipping test that is not gcp only")
	}

	Run(t, "testdata/dev", "sh", "-c", "echo foo > foo")
	defer Run(t, "testdata/dev", "rm", "foo")

	// Run skaffold build first to fail quickly on a build failure
	skaffold.Build().InDir("testdata/dev").RunOrFail(t)

	ns, k8sClient, deleteNs := SetupNamespace(t)
	defer deleteNs()

	rpcAddr := randomPort()

	stop := skaffold.Dev("--auto-build=false", "--auto-sync=false", "--auto-deploy=false", "--rpc-port", rpcAddr, "--cache-artifacts=false").InDir("testdata/dev").InNs(ns.Name).RunBackground(t)
	defer stop()

	client, shutdown := setupRPCClient(t, rpcAddr)
	defer shutdown()

	stream, err := readEventAPIStream(client, t, readRetries)
	if stream == nil {
		t.Fatalf("error retrieving event log: %v\n", err)
	}

	// throw away first 5 entries of log (from first run of dev loop)
	for i := 0; i < 5; i++ {
		stream.Recv()
	}

	// read entries from the log
	entries := make(chan *proto.LogEntry)
	go func() {
		for {
			entry, _ := stream.Recv()
			if entry != nil {
				entries <- entry
			}
		}
	}()

	dep := k8sClient.GetDeployment("test-dev")

	// Make a change to foo
	Run(t, "testdata/dev", "sh", "-c", "echo bar > foo")

	// Issue a build trigger
	client.Execute(context.Background(), &proto.UserIntentRequest{
		Intent: &proto.Intent{
			Build: true,
		},
	})

	// Ensure we see a build triggered in the event log
	err = wait.PollImmediate(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
		e := <-entries
		return e.GetEvent().GetBuildEvent().GetArtifact() == "gcr.io/k8s-skaffold/test-dev", nil
	})
	testutil.CheckError(t, false, err)

	// Issue a deploy trigger
	client.Execute(context.Background(), &proto.UserIntentRequest{
		Intent: &proto.Intent{
			Deploy: true,
		},
	})

	// Ensure we see a deploy triggered in the event log
	err = wait.PollImmediate(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
		e := <-entries
		return e.GetEvent().GetDeployEvent().GetStatus() == "In Progress", nil
	})
	testutil.CheckError(t, false, err)

	// Make sure the old Deployment and the new Deployment are different
	err = wait.PollImmediate(time.Millisecond*500, 10*time.Minute, func() (bool, error) {
		newDep := k8sClient.GetDeployment("test-dev")
		return dep.GetGeneration() != newDep.GetGeneration(), nil
	})
	testutil.CheckError(t, false, err)
}

func TestDevPortForward(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test")
	}
	if ShouldRunGCPOnlyTests() {
		t.Skip("skipping test that is not gcp only")
	}

	// Run skaffold build first to fail quickly on a build failure
	skaffold.Build("--cache-artifacts=true").InDir("examples/microservices").RunOrFail(t)

	ns, _, deleteNs := SetupNamespace(t)
	defer deleteNs()

	rpcAddr := randomPort()
	env := []string{fmt.Sprintf("TEST_NS=%s", ns.Name)}
	cmd := skaffold.Dev("--port-forward", "--rpc-port", rpcAddr, "--cache-artifacts=true").InDir("examples/microservices").InNs(ns.Name).WithEnv(env)
	stop := cmd.RunBackground(t)
	defer stop()

	client, shutdown := setupRPCClient(t, rpcAddr)
	defer shutdown()

	// create a grpc connection. Increase number of reties for helm.
	stream, err := readEventAPIStream(client, t, 20)
	if stream == nil {
		t.Fatalf("error retrieving event log: %v\n", err)
	}

	// read entries from the log
	entries := make(chan *proto.LogEntry)
	go func() {
		for {
			entry, _ := stream.Recv()
			if entry != nil {
				entries <- entry
			}
		}
	}()

	originalResponse := "leeroooooy app!!"
	replacementResponse := "test string"

	waitForPortForwardEvent(t, entries, "leeroy-app", "service", ns.Name, originalResponse+"\n")

	original, perms, fErr := replaceInFile(originalResponse, replacementResponse, "examples/microservices/leeroy-app/app.go")
	if fErr != nil {
		t.Error(fErr)
	}
	defer func() {
		if original != nil {
			ioutil.WriteFile("examples/microservices/leeroy-app/app.go", original, perms)
		}
	}()

	waitForPortForwardEvent(t, entries, "leeroy-app", "service", ns.Name, replacementResponse+"\n")
}

func TestDevPortForwardGKELoadBalancer(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test")
	}
	if !ShouldRunGCPOnlyTests() {
		t.Skip("skipping test that is not gcp only")
	}

	// Run skaffold build first to fail quickly on a build failure
	skaffold.Build().InDir("testdata/gke_loadbalancer").RunOrFail(t)

	ns, _, deleteNs := SetupNamespace(t)
	defer deleteNs()

	rpcAddr := randomPort()
	env := []string{fmt.Sprintf("TEST_NS=%s", ns.Name)}
	cmd := skaffold.Dev("--port-forward", "--rpc-port", rpcAddr).InDir("testdata/gke_loadbalancer").InNs(ns.Name).WithEnv(env)
	stop := cmd.RunBackground(t)
	defer stop()

	client, shutdown := setupRPCClient(t, rpcAddr)
	defer shutdown()

	// create a grpc connection. Increase number of reties for helm.
	stream, err := readEventAPIStream(client, t, 20)
	if stream == nil {
		t.Fatalf("error retrieving event log: %v\n", err)
	}

	// read entries from the log
	entries := make(chan *proto.LogEntry)
	go func() {
		for {
			entry, _ := stream.Recv()
			if entry != nil {
				entries <- entry
			}
		}
	}()

	waitForPortForwardEvent(t, entries, "gke-loadbalancer", "service", ns.Name, "hello!!\n")
}

func getLocalPortFromPortForwardEvent(t *testing.T, entries chan *proto.LogEntry, resourceName, resourceType, namespace string) int {
	timeout := time.After(1 * time.Minute)
	for {
		select {
		case <-timeout:
			t.Fatalf("timed out waiting for port forwarding event")
		case e := <-entries:
			switch e.Event.GetEventType().(type) {
			case *proto.Event_PortEvent:
				t.Logf("event received %v", e)
				if e.Event.GetPortEvent().ResourceName == resourceName &&
					e.Event.GetPortEvent().ResourceType == resourceType &&
					e.Event.GetPortEvent().Namespace == namespace {
					port := e.Event.GetPortEvent().LocalPort
					t.Logf("Detected %s/%s is forwarded to port %d", resourceType, resourceName, port)
					return int(port)
				}
			default:
				t.Logf("event received %v", e)
			}
		}
	}
}

func waitForPortForwardEvent(t *testing.T, entries chan *proto.LogEntry, resourceName, resourceType, namespace, expected string) {
	port := getLocalPortFromPortForwardEvent(t, entries, resourceName, resourceType, namespace)
	assertResponseFromPort(t, port, expected)
}

// assertResponseFromPort waits for two minutes for the expected response at port.
func assertResponseFromPort(t *testing.T, port int, expected string) {
	logrus.Infof("Waiting for response %s from port %d", expected, port)
	ctx, cancelTimeout := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancelTimeout()

	for {
		select {
		case <-ctx.Done():
			t.Fatalf("Timed out waiting for response from port %d", port)
		case <-time.After(1 * time.Second):
			resp, err := http.Get(fmt.Sprintf("http://%s:%d", util.Loopback, port))
			if err != nil {
				logrus.Infof("error getting response from port %d: %v", port, err)
				continue
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				logrus.Infof("error reading response: %v", err)
				continue
			}
			if string(body) == expected {
				return
			}
			logrus.Infof("didn't get expected response from port. got: %s, expected: %s", string(body), expected)
		}
	}
}

func replaceInFile(target, replacement, filepath string) ([]byte, os.FileMode, error) {
	fInfo, err := os.Stat(filepath)
	if err != nil {
		return nil, 0, err
	}
	original, err := ioutil.ReadFile(filepath)
	if err != nil {
		return nil, 0, err
	}

	newContents := strings.Replace(string(original), target, replacement, -1)

	err = ioutil.WriteFile(filepath, []byte(newContents), 0)

	return original, fInfo.Mode(), err
}

func readEventAPIStream(client proto.SkaffoldServiceClient, t *testing.T, retries int) (proto.SkaffoldService_EventLogClient, error) {
	t.Helper()
	// read the event log stream from the skaffold grpc server
	var stream proto.SkaffoldService_EventLogClient
	var err error
	for i := 0; i < retries; i++ {
		stream, err = client.EventLog(context.Background())
		if err != nil {
			t.Logf("waiting for connection...")
			time.Sleep(waitTime)
			continue
		}
	}
	return stream, err
}

func TestDev_WithKubecontextOverride(t *testing.T) {
	const (
		kubecontext = "modified-context"
		kubeconfig  = "kubeconfig"
	)

	if testing.Short() {
		t.Skip("skipping integration test")
	}

	testutil.Run(t, "skaffold run with kubecontext override", func(t *testutil.T) {
		dir := "examples/getting-started"
		pods := []string{"getting-started"}

		ns, client, deleteNs := SetupNamespace(t.T)
		defer deleteNs()

		modifiedKubeconfig, err := createModifiedKubeconfig(kubecontext, ns.Name)
		if err != nil {
			t.Fatal(err)
		}
		kubeconfig := t.NewTempDir().
			Write(kubeconfig, string(modifiedKubeconfig)).
			Path(kubeconfig)
		env := []string{fmt.Sprintf("KUBECONFIG=%s", kubeconfig)}

		// n.b. for the sake of this test the namespace must not be given explicitly
		skaffold.Run("--kube-context", kubecontext).InDir(dir).WithEnv(env).RunOrFailOutput(t.T)

		client.WaitForPodsReady(pods...)

		// n.b. for the sake of this test the namespace must not be given explicitly
		skaffold.Delete("--kube-context", kubecontext).InDir(dir).WithEnv(env).RunOrFail(t.T)
	})
}

func createModifiedKubeconfig(contextName, namespace string) ([]byte, error) {
	config, err := clientcmd.NewDefaultClientConfigLoadingRules().Load()
	if err != nil {
		return nil, err
	}
	activeContext := config.Contexts[config.CurrentContext]
	// clear the namespace in the active context
	activeContext.Namespace = ""

	newContext := activeContext.DeepCopy()
	newContext.Namespace = namespace
	config.Contexts[contextName] = newContext

	return clientcmd.Write(*config)
}
<commit_msg>Use different context name when running on kind<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	kubectx "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/context"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/proto"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestDev(t *testing.T) {
	tests := []struct {
		description string
		trigger     string
	}{
		{
			description: "dev with polling trigger",
			trigger:     "polling",
		},
		{
			description: "dev with notify trigger",
			trigger:     "notify",
		},
	}
	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			if testing.Short() {
				t.Skip("skipping integration test")
			}
			if ShouldRunGCPOnlyTests() {
				t.Skip("skipping test that is not gcp only")
			}

			Run(t, "testdata/dev", "sh", "-c", "echo foo > foo")
			defer Run(t, "testdata/dev", "rm", "foo")

			// Run skaffold build first to fail quickly on a build failure
			skaffold.Build().InDir("testdata/dev").RunOrFail(t)

			ns, client, deleteNs := SetupNamespace(t)
			defer deleteNs()

			stop := skaffold.Dev("--trigger", test.trigger).InDir("testdata/dev").InNs(ns.Name).RunBackground(t)
			defer stop()

			dep := client.GetDeployment("test-dev")

			// Make a change to foo so that dev is forced to delete the Deployment and redeploy
			Run(t, "testdata/dev", "sh", "-c", "echo bar > foo")

			// Make sure the old Deployment and the new Deployment are different
			err := wait.PollImmediate(time.Millisecond*500, 10*time.Minute, func() (bool, error) {
				newDep := client.GetDeployment("test-dev")
				return dep.GetGeneration() != newDep.GetGeneration(), nil
			})
			testutil.CheckError(t, false, err)
		})
	}
}

func TestDevAPITriggers(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test")
	}
	if ShouldRunGCPOnlyTests() {
		t.Skip("skipping test that is not gcp only")
	}

	Run(t, "testdata/dev", "sh", "-c", "echo foo > foo")
	defer Run(t, "testdata/dev", "rm", "foo")

	// Run skaffold build first to fail quickly on a build failure
	skaffold.Build().InDir("testdata/dev").RunOrFail(t)

	ns, k8sClient, deleteNs := SetupNamespace(t)
	defer deleteNs()

	rpcAddr := randomPort()

	stop := skaffold.Dev("--auto-build=false", "--auto-sync=false", "--auto-deploy=false", "--rpc-port", rpcAddr, "--cache-artifacts=false").InDir("testdata/dev").InNs(ns.Name).RunBackground(t)
	defer stop()

	client, shutdown := setupRPCClient(t, rpcAddr)
	defer shutdown()

	stream, err := readEventAPIStream(client, t, readRetries)
	if stream == nil {
		t.Fatalf("error retrieving event log: %v\n", err)
	}

	// throw away first 5 entries of log (from first run of dev loop)
	for i := 0; i < 5; i++ {
		stream.Recv()
	}

	// read entries from the log
	entries := make(chan *proto.LogEntry)
	go func() {
		for {
			entry, _ := stream.Recv()
			if entry != nil {
				entries <- entry
			}
		}
	}()

	dep := k8sClient.GetDeployment("test-dev")

	// Make a change to foo
	Run(t, "testdata/dev", "sh", "-c", "echo bar > foo")

	// Issue a build trigger
	client.Execute(context.Background(), &proto.UserIntentRequest{
		Intent: &proto.Intent{
			Build: true,
		},
	})

	// Ensure we see a build triggered in the event log
	err = wait.PollImmediate(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
		e := <-entries
		return e.GetEvent().GetBuildEvent().GetArtifact() == "gcr.io/k8s-skaffold/test-dev", nil
	})
	testutil.CheckError(t, false, err)

	// Issue a deploy trigger
	client.Execute(context.Background(), &proto.UserIntentRequest{
		Intent: &proto.Intent{
			Deploy: true,
		},
	})

	// Ensure we see a deploy triggered in the event log
	err = wait.PollImmediate(time.Millisecond*500, 2*time.Minute, func() (bool, error) {
		e := <-entries
		return e.GetEvent().GetDeployEvent().GetStatus() == "In Progress", nil
	})
	testutil.CheckError(t, false, err)

	// Make sure the old Deployment and the new Deployment are different
	err = wait.PollImmediate(time.Millisecond*500, 10*time.Minute, func() (bool, error) {
		newDep := k8sClient.GetDeployment("test-dev")
		return dep.GetGeneration() != newDep.GetGeneration(), nil
	})
	testutil.CheckError(t, false, err)
}

func TestDevPortForward(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test")
	}
	if ShouldRunGCPOnlyTests() {
		t.Skip("skipping test that is not gcp only")
	}

	// Run skaffold build first to fail quickly on a build failure
	skaffold.Build("--cache-artifacts=true").InDir("examples/microservices").RunOrFail(t)

	ns, _, deleteNs := SetupNamespace(t)
	defer deleteNs()

	rpcAddr := randomPort()
	env := []string{fmt.Sprintf("TEST_NS=%s", ns.Name)}
	cmd := skaffold.Dev("--port-forward", "--rpc-port", rpcAddr, "--cache-artifacts=true").InDir("examples/microservices").InNs(ns.Name).WithEnv(env)
	stop := cmd.RunBackground(t)
	defer stop()

	client, shutdown := setupRPCClient(t, rpcAddr)
	defer shutdown()

	// create a grpc connection. Increase number of reties for helm.
	stream, err := readEventAPIStream(client, t, 20)
	if stream == nil {
		t.Fatalf("error retrieving event log: %v\n", err)
	}

	// read entries from the log
	entries := make(chan *proto.LogEntry)
	go func() {
		for {
			entry, _ := stream.Recv()
			if entry != nil {
				entries <- entry
			}
		}
	}()

	originalResponse := "leeroooooy app!!"
	replacementResponse := "test string"

	waitForPortForwardEvent(t, entries, "leeroy-app", "service", ns.Name, originalResponse+"\n")

	original, perms, fErr := replaceInFile(originalResponse, replacementResponse, "examples/microservices/leeroy-app/app.go")
	if fErr != nil {
		t.Error(fErr)
	}
	defer func() {
		if original != nil {
			ioutil.WriteFile("examples/microservices/leeroy-app/app.go", original, perms)
		}
	}()

	waitForPortForwardEvent(t, entries, "leeroy-app", "service", ns.Name, replacementResponse+"\n")
}

func TestDevPortForwardGKELoadBalancer(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test")
	}
	if !ShouldRunGCPOnlyTests() {
		t.Skip("skipping test that is not gcp only")
	}

	// Run skaffold build first to fail quickly on a build failure
	skaffold.Build().InDir("testdata/gke_loadbalancer").RunOrFail(t)

	ns, _, deleteNs := SetupNamespace(t)
	defer deleteNs()

	rpcAddr := randomPort()
	env := []string{fmt.Sprintf("TEST_NS=%s", ns.Name)}
	cmd := skaffold.Dev("--port-forward", "--rpc-port", rpcAddr).InDir("testdata/gke_loadbalancer").InNs(ns.Name).WithEnv(env)
	stop := cmd.RunBackground(t)
	defer stop()

	client, shutdown := setupRPCClient(t, rpcAddr)
	defer shutdown()

	// create a grpc connection. Increase number of reties for helm.
	stream, err := readEventAPIStream(client, t, 20)
	if stream == nil {
		t.Fatalf("error retrieving event log: %v\n", err)
	}

	// read entries from the log
	entries := make(chan *proto.LogEntry)
	go func() {
		for {
			entry, _ := stream.Recv()
			if entry != nil {
				entries <- entry
			}
		}
	}()

	waitForPortForwardEvent(t, entries, "gke-loadbalancer", "service", ns.Name, "hello!!\n")
}

func getLocalPortFromPortForwardEvent(t *testing.T, entries chan *proto.LogEntry, resourceName, resourceType, namespace string) int {
	timeout := time.After(1 * time.Minute)
	for {
		select {
		case <-timeout:
			t.Fatalf("timed out waiting for port forwarding event")
		case e := <-entries:
			switch e.Event.GetEventType().(type) {
			case *proto.Event_PortEvent:
				t.Logf("event received %v", e)
				if e.Event.GetPortEvent().ResourceName == resourceName &&
					e.Event.GetPortEvent().ResourceType == resourceType &&
					e.Event.GetPortEvent().Namespace == namespace {
					port := e.Event.GetPortEvent().LocalPort
					t.Logf("Detected %s/%s is forwarded to port %d", resourceType, resourceName, port)
					return int(port)
				}
			default:
				t.Logf("event received %v", e)
			}
		}
	}
}

func waitForPortForwardEvent(t *testing.T, entries chan *proto.LogEntry, resourceName, resourceType, namespace, expected string) {
	port := getLocalPortFromPortForwardEvent(t, entries, resourceName, resourceType, namespace)
	assertResponseFromPort(t, port, expected)
}

// assertResponseFromPort waits for two minutes for the expected response at port.
func assertResponseFromPort(t *testing.T, port int, expected string) {
	logrus.Infof("Waiting for response %s from port %d", expected, port)
	ctx, cancelTimeout := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancelTimeout()

	for {
		select {
		case <-ctx.Done():
			t.Fatalf("Timed out waiting for response from port %d", port)
		case <-time.After(1 * time.Second):
			resp, err := http.Get(fmt.Sprintf("http://%s:%d", util.Loopback, port))
			if err != nil {
				logrus.Infof("error getting response from port %d: %v", port, err)
				continue
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				logrus.Infof("error reading response: %v", err)
				continue
			}
			if string(body) == expected {
				return
			}
			logrus.Infof("didn't get expected response from port. got: %s, expected: %s", string(body), expected)
		}
	}
}

func replaceInFile(target, replacement, filepath string) ([]byte, os.FileMode, error) {
	fInfo, err := os.Stat(filepath)
	if err != nil {
		return nil, 0, err
	}
	original, err := ioutil.ReadFile(filepath)
	if err != nil {
		return nil, 0, err
	}

	newContents := strings.Replace(string(original), target, replacement, -1)

	err = ioutil.WriteFile(filepath, []byte(newContents), 0)

	return original, fInfo.Mode(), err
}

func readEventAPIStream(client proto.SkaffoldServiceClient, t *testing.T, retries int) (proto.SkaffoldService_EventLogClient, error) {
	t.Helper()
	// read the event log stream from the skaffold grpc server
	var stream proto.SkaffoldService_EventLogClient
	var err error
	for i := 0; i < retries; i++ {
		stream, err = client.EventLog(context.Background())
		if err != nil {
			t.Logf("waiting for connection...")
			time.Sleep(waitTime)
			continue
		}
	}
	return stream, err
}

func TestDev_WithKubecontextOverride(t *testing.T) {
	const (
		kubeconfig = "kubeconfig"
	)

	if testing.Short() {
		t.Skip("skipping integration test")
	}

	testutil.Run(t, "skaffold run with kubecontext override", func(t *testutil.T) {
		dir := "examples/getting-started"
		pods := []string{"getting-started"}

		ns, client, deleteNs := SetupNamespace(t.T)
		defer deleteNs()

		modifiedKubeconfig, kubecontext, err := createModifiedKubeconfig(ns.Name)
		if err != nil {
			t.Fatal(err)
		}
		kubeconfig := t.NewTempDir().
			Write(kubeconfig, string(modifiedKubeconfig)).
			Path(kubeconfig)
		env := []string{fmt.Sprintf("KUBECONFIG=%s", kubeconfig)}

		// n.b. for the sake of this test the namespace must not be given explicitly
		skaffold.Run("--kube-context", kubecontext).InDir(dir).WithEnv(env).RunOrFailOutput(t.T)

		client.WaitForPodsReady(pods...)

		// n.b. for the sake of this test the namespace must not be given explicitly
		skaffold.Delete("--kube-context", kubecontext).InDir(dir).WithEnv(env).RunOrFail(t.T)
	})
}

func createModifiedKubeconfig(namespace string) ([]byte, string, error) {
	kubeConfig, err := kubectx.CurrentConfig()
	if err != nil {
		return nil, "", err
	}

	contextName := "modified-context"
	if config.IsKindCluster(kubeConfig.CurrentContext) {
		contextName = contextName + "@kind"
	}

	activeContext := kubeConfig.Contexts[kubeConfig.CurrentContext]
	// clear the namespace in the active context
	activeContext.Namespace = ""

	newContext := activeContext.DeepCopy()
	newContext.Namespace = namespace
	kubeConfig.Contexts[contextName] = newContext

	yaml, err := clientcmd.Write(kubeConfig)
	return yaml, contextName, err
}
<|endoftext|>


<commit_before>// Copyright 2015 - 2017 Ka-Hing Cheung
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"io"
	"runtime"
	"runtime/debug"
	"sync"

	"github.com/jacobsa/fuse"
	"github.com/shirou/gopsutil/mem"
)

type BufferPool struct {
	mu   sync.Mutex
	cond *sync.Cond

	numBuffers uint64
	maxBuffers uint64

	totalBuffers       uint64
	computedMaxbuffers uint64

	pool *sync.Pool
}

const BUF_SIZE = 5 * 1024 * 1024

func maxMemToUse(buffersNow uint64) uint64 {
	m, err := mem.VirtualMemory()
	if err != nil {
		panic(err)
	}

	availableMem, err := getCgroupAvailableMem()
	log.Debugf("amount of available memory from cgroup is: %v", availableMem/1024/1024)

	if err != nil || availableMem < 0 || availableMem > m.Available {
		availableMem = m.Available
	}

	log.Debugf("amount of available memory: %v", availableMem/1024/1024)

	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)

	log.Debugf("amount of allocated memory: %v %v", ms.Sys/1024/1024, ms.Alloc/1024/1024)

	max := uint64(availableMem+ms.Sys) / 2
	maxbuffers := MaxUInt64(max/BUF_SIZE, 1)
	log.Debugf("using up to %v %vMB buffers, now is %v", maxbuffers, BUF_SIZE/1024/1024, buffersNow)
	return maxbuffers
}

func rounduUp(size uint64, pageSize int) int {
	return pages(size, pageSize) * pageSize
}

func pages(size uint64, pageSize int) int {
	return int((size + uint64(pageSize) - 1) / uint64(pageSize))
}

func (pool BufferPool) Init() *BufferPool {
	pool.cond = sync.NewCond(&pool.mu)

	pool.computedMaxbuffers = pool.maxBuffers
	pool.pool = &sync.Pool{New: func() interface{} {
		return make([]byte, 0, BUF_SIZE)
	}}

	return &pool
}

// for testing
func NewBufferPool(maxSizeGlobal uint64) *BufferPool {
	pool := BufferPool{maxBuffers: maxSizeGlobal / BUF_SIZE}.Init()
	return pool
}

func (pool *BufferPool) RequestBuffer() (buf []byte) {
	return pool.RequestMultiple(BUF_SIZE, true)[0]
}

func (pool *BufferPool) recomputeBufferLimit() {
	if pool.maxBuffers == 0 {
		pool.computedMaxbuffers = maxMemToUse(pool.numBuffers)
		if pool.computedMaxbuffers == 0 {
			panic("OOM")
		}
	}
}

func (pool *BufferPool) RequestMultiple(size uint64, block bool) (buffers [][]byte) {
	nPages := pages(size, BUF_SIZE)

	pool.mu.Lock()
	defer pool.mu.Unlock()

	if pool.totalBuffers%10 == 0 {
		pool.recomputeBufferLimit()
	}

	for pool.numBuffers+uint64(nPages) > pool.computedMaxbuffers {
		if block {
			if pool.numBuffers == 0 {
				pool.MaybeGC()
				pool.recomputeBufferLimit()
				if pool.numBuffers+uint64(nPages) > pool.computedMaxbuffers {
					// we don't have any in use buffers, and we've made attempts to
					// free memory AND correct our limits, yet we still can't allocate.
					// it's likely that we are simply asking for too much
					log.Errorf("Unable to allocate %d bytes, limit is %d bytes",
						nPages*BUF_SIZE, pool.computedMaxbuffers*BUF_SIZE)
					panic("OOM")
				}
			}
			pool.cond.Wait()
		} else {
			return
		}
	}

	for i := 0; i < nPages; i++ {
		pool.numBuffers++
		pool.totalBuffers++
		buf := pool.pool.Get()
		buffers = append(buffers, buf.([]byte))
	}
	return
}

func (pool *BufferPool) MaybeGC() {
	if pool.numBuffers == 0 {
		debug.FreeOSMemory()
	}
}

func (pool *BufferPool) Free(buf []byte) {
	pool.mu.Lock()
	defer pool.mu.Unlock()

	buf = buf[:0]
	pool.pool.Put(buf)
	pool.numBuffers--
	pool.cond.Signal()
}

var mbufLog = GetLogger("mbuf")

type MBuf struct {
	pool    *BufferPool
	buffers [][]byte
	rbuf    int
	wbuf    int
	rp      int
	wp      int
}

func (mb MBuf) Init(h *BufferPool, size uint64, block bool) *MBuf {
	mb.pool = h

	if size != 0 {
		mb.buffers = h.RequestMultiple(size, block)
		if mb.buffers == nil {
			return nil
		}
	}

	return &mb
}

func (mb *MBuf) Len() (length int) {
	for i := mb.rbuf; i < int(len(mb.buffers)); i++ {
		var bufSize int
		var start int
		if i == mb.wbuf {
			bufSize = mb.wp
		} else {
			bufSize = int(len(mb.buffers[i]))
		}

		if i == mb.rbuf {
			start = mb.rp
		} else {
			start = 0
		}

		length += bufSize - start
	}

	return
}

// seek only seeks the reader
func (mb *MBuf) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case 0: // relative to beginning
		if offset == 0 {
			mb.rbuf = 0
			mb.rp = 0
			return 0, nil
		}
	case 1: // relative to current position
		if offset == 0 {
			for i := 0; i < mb.rbuf; i++ {
				offset += int64(len(mb.buffers[i]))
			}
			offset += int64(mb.rp)
			return offset, nil
		}

	case 2: // relative to the end
		if offset == 0 {
			for i := 0; i < len(mb.buffers); i++ {
				offset += int64(len(mb.buffers[i]))
			}
			mb.rbuf = len(mb.buffers)
			mb.rp = 0
			return offset, nil
		}
	}

	log.Errorf("Seek %d %d", offset, whence)
	panic(fuse.EINVAL)

	return 0, fuse.EINVAL
}

func (mb *MBuf) Read(p []byte) (n int, err error) {
	if mb.rbuf == mb.wbuf && mb.rp == mb.wp {
		err = io.EOF
		return
	}

	if mb.rp == cap(mb.buffers[mb.rbuf]) {
		mb.rbuf++
		mb.rp = 0
	}

	if mb.rbuf == len(mb.buffers) {
		err = io.EOF
		return
	} else if mb.rbuf > len(mb.buffers) {
		panic("mb.cur > len(mb.buffers)")
	}

	n = copy(p, mb.buffers[mb.rbuf][mb.rp:])
	mb.rp += n

	return
}

func (mb *MBuf) Full() bool {
	return mb.buffers == nil || (mb.wp == cap(mb.buffers[mb.wbuf]) && mb.wbuf+1 == len(mb.buffers))
}

func (mb *MBuf) Write(p []byte) (n int, err error) {
	b := mb.buffers[mb.wbuf]

	if mb.wp == cap(b) {
		if mb.wbuf+1 == len(mb.buffers) {
			return
		}
		mb.wbuf++
		b = mb.buffers[mb.wbuf]
		mb.wp = 0
	} else if mb.wp > cap(b) {
		panic("mb.wp > cap(b)")
	}

	n = copy(b[mb.wp:cap(b)], p)
	mb.wp += n
	// resize the buffer to account for what we just read
	mb.buffers[mb.wbuf] = mb.buffers[mb.wbuf][:mb.wp]

	return
}

func (mb *MBuf) WriteFrom(r io.Reader) (n int, err error) {
	b := mb.buffers[mb.wbuf]

	if mb.wp == cap(b) {
		if mb.wbuf+1 == len(mb.buffers) {
			return
		}
		mb.wbuf++
		b = mb.buffers[mb.wbuf]
		mb.wp = 0
	} else if mb.wp > cap(b) {
		panic("mb.wp > cap(b)")
	}

	n, err = r.Read(b[mb.wp:cap(b)])
	mb.wp += n
	// resize the buffer to account for what we just read
	mb.buffers[mb.wbuf] = mb.buffers[mb.wbuf][:mb.wp]

	return
}

func (mb *MBuf) Free() {
	for _, b := range mb.buffers {
		mb.pool.Free(b)
	}

	mb.buffers = nil
}

var bufferLog = GetLogger("buffer")

type Buffer struct {
	mu   sync.Mutex
	cond *sync.Cond

	buf    *MBuf
	reader io.ReadCloser
	err    error
}

type ReaderProvider func() (io.ReadCloser, error)

func (b Buffer) Init(buf *MBuf, r ReaderProvider) *Buffer {
	b.buf = buf
	b.cond = sync.NewCond(&b.mu)

	go func() {
		b.readLoop(r)
	}()

	return &b
}

func (b *Buffer) readLoop(r ReaderProvider) {
	for {
		b.mu.Lock()
		if b.reader == nil {
			b.reader, b.err = r()
			b.cond.Broadcast()
			if b.err != nil {
				b.mu.Unlock()
				break
			}
		}

		if b.buf == nil {
			// buffer was drained
			b.mu.Unlock()
			break
		}

		nread, err := b.buf.WriteFrom(b.reader)
		if err != nil {
			b.err = err
			b.mu.Unlock()
			break
		}
		bufferLog.Debugf("wrote %v into buffer", nread)

		if nread == 0 {
			b.reader.Close()
			b.mu.Unlock()
			break
		}

		b.mu.Unlock()
		// if we get here we've read _something_, bounce this goroutine
		// to allow another one to read
		runtime.Gosched()
	}
	bufferLog.Debugf("<-- readLoop()")
}

func (b *Buffer) readFromStream(p []byte) (n int, err error) {
	bufferLog.Debugf("reading %v from stream", len(p))

	n, err = b.reader.Read(p)
	if n != 0 && err == io.ErrUnexpectedEOF {
		err = nil
	} else {
		bufferLog.Debugf("read %v from stream", n)
	}
	return
}

func (b *Buffer) Read(p []byte) (n int, err error) {
	bufferLog.Debugf("Buffer.Read(%v)", len(p))

	b.mu.Lock()
	defer b.mu.Unlock()

	for b.reader == nil && b.err == nil {
		bufferLog.Debugf("waiting for stream")
		b.cond.Wait()
		if b.err != nil {
			err = b.err
			return
		}
	}

	// we could have received the err before Read was called
	if b.reader == nil {
		if b.err == nil {
			panic("reader and err are both nil")
		}
		err = b.err
		return
	}

	if b.buf != nil {
		bufferLog.Debugf("reading %v from buffer", len(p))

		n, err = b.buf.Read(p)
		if n == 0 {
			b.buf.Free()
			b.buf = nil
			bufferLog.Debugf("drained buffer")
			n, err = b.readFromStream(p)
		} else {
			bufferLog.Debugf("read %v from buffer", n)
		}
	} else if b.err != nil {
		err = b.err
	} else {
		n, err = b.readFromStream(p)
	}

	return
}

func (b *Buffer) Close() (err error) {
	b.mu.Lock()
	defer b.mu.Unlock()

	if b.reader != nil {
		err = b.reader.Close()
	}

	if b.buf != nil {
		b.buf.Free()
		b.buf = nil
	}

	return
}
<commit_msg>only log if we can get it from cgroup<commit_after>// Copyright 2015 - 2017 Ka-Hing Cheung
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"io"
	"runtime"
	"runtime/debug"
	"sync"

	"github.com/jacobsa/fuse"
	"github.com/shirou/gopsutil/mem"
)

type BufferPool struct {
	mu   sync.Mutex
	cond *sync.Cond

	numBuffers uint64
	maxBuffers uint64

	totalBuffers       uint64
	computedMaxbuffers uint64

	pool *sync.Pool
}

const BUF_SIZE = 5 * 1024 * 1024

func maxMemToUse(buffersNow uint64) uint64 {
	m, err := mem.VirtualMemory()
	if err != nil {
		panic(err)
	}

	availableMem, err := getCgroupAvailableMem()
	if err != nil {
		log.Debugf("amount of available memory from cgroup is: %v", availableMem/1024/1024)
	}

	if err != nil || availableMem < 0 || availableMem > m.Available {
		availableMem = m.Available
	}

	log.Debugf("amount of available memory: %v", availableMem/1024/1024)

	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)

	log.Debugf("amount of allocated memory: %v %v", ms.Sys/1024/1024, ms.Alloc/1024/1024)

	max := uint64(availableMem+ms.Sys) / 2
	maxbuffers := MaxUInt64(max/BUF_SIZE, 1)
	log.Debugf("using up to %v %vMB buffers, now is %v", maxbuffers, BUF_SIZE/1024/1024, buffersNow)
	return maxbuffers
}

func rounduUp(size uint64, pageSize int) int {
	return pages(size, pageSize) * pageSize
}

func pages(size uint64, pageSize int) int {
	return int((size + uint64(pageSize) - 1) / uint64(pageSize))
}

func (pool BufferPool) Init() *BufferPool {
	pool.cond = sync.NewCond(&pool.mu)

	pool.computedMaxbuffers = pool.maxBuffers
	pool.pool = &sync.Pool{New: func() interface{} {
		return make([]byte, 0, BUF_SIZE)
	}}

	return &pool
}

// for testing
func NewBufferPool(maxSizeGlobal uint64) *BufferPool {
	pool := BufferPool{maxBuffers: maxSizeGlobal / BUF_SIZE}.Init()
	return pool
}

func (pool *BufferPool) RequestBuffer() (buf []byte) {
	return pool.RequestMultiple(BUF_SIZE, true)[0]
}

func (pool *BufferPool) recomputeBufferLimit() {
	if pool.maxBuffers == 0 {
		pool.computedMaxbuffers = maxMemToUse(pool.numBuffers)
		if pool.computedMaxbuffers == 0 {
			panic("OOM")
		}
	}
}

func (pool *BufferPool) RequestMultiple(size uint64, block bool) (buffers [][]byte) {
	nPages := pages(size, BUF_SIZE)

	pool.mu.Lock()
	defer pool.mu.Unlock()

	if pool.totalBuffers%10 == 0 {
		pool.recomputeBufferLimit()
	}

	for pool.numBuffers+uint64(nPages) > pool.computedMaxbuffers {
		if block {
			if pool.numBuffers == 0 {
				pool.MaybeGC()
				pool.recomputeBufferLimit()
				if pool.numBuffers+uint64(nPages) > pool.computedMaxbuffers {
					// we don't have any in use buffers, and we've made attempts to
					// free memory AND correct our limits, yet we still can't allocate.
					// it's likely that we are simply asking for too much
					log.Errorf("Unable to allocate %d bytes, limit is %d bytes",
						nPages*BUF_SIZE, pool.computedMaxbuffers*BUF_SIZE)
					panic("OOM")
				}
			}
			pool.cond.Wait()
		} else {
			return
		}
	}

	for i := 0; i < nPages; i++ {
		pool.numBuffers++
		pool.totalBuffers++
		buf := pool.pool.Get()
		buffers = append(buffers, buf.([]byte))
	}
	return
}

func (pool *BufferPool) MaybeGC() {
	if pool.numBuffers == 0 {
		debug.FreeOSMemory()
	}
}

func (pool *BufferPool) Free(buf []byte) {
	pool.mu.Lock()
	defer pool.mu.Unlock()

	buf = buf[:0]
	pool.pool.Put(buf)
	pool.numBuffers--
	pool.cond.Signal()
}

var mbufLog = GetLogger("mbuf")

type MBuf struct {
	pool    *BufferPool
	buffers [][]byte
	rbuf    int
	wbuf    int
	rp      int
	wp      int
}

func (mb MBuf) Init(h *BufferPool, size uint64, block bool) *MBuf {
	mb.pool = h

	if size != 0 {
		mb.buffers = h.RequestMultiple(size, block)
		if mb.buffers == nil {
			return nil
		}
	}

	return &mb
}

func (mb *MBuf) Len() (length int) {
	for i := mb.rbuf; i < int(len(mb.buffers)); i++ {
		var bufSize int
		var start int
		if i == mb.wbuf {
			bufSize = mb.wp
		} else {
			bufSize = int(len(mb.buffers[i]))
		}

		if i == mb.rbuf {
			start = mb.rp
		} else {
			start = 0
		}

		length += bufSize - start
	}

	return
}

// seek only seeks the reader
func (mb *MBuf) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case 0: // relative to beginning
		if offset == 0 {
			mb.rbuf = 0
			mb.rp = 0
			return 0, nil
		}
	case 1: // relative to current position
		if offset == 0 {
			for i := 0; i < mb.rbuf; i++ {
				offset += int64(len(mb.buffers[i]))
			}
			offset += int64(mb.rp)
			return offset, nil
		}

	case 2: // relative to the end
		if offset == 0 {
			for i := 0; i < len(mb.buffers); i++ {
				offset += int64(len(mb.buffers[i]))
			}
			mb.rbuf = len(mb.buffers)
			mb.rp = 0
			return offset, nil
		}
	}

	log.Errorf("Seek %d %d", offset, whence)
	panic(fuse.EINVAL)

	return 0, fuse.EINVAL
}

func (mb *MBuf) Read(p []byte) (n int, err error) {
	if mb.rbuf == mb.wbuf && mb.rp == mb.wp {
		err = io.EOF
		return
	}

	if mb.rp == cap(mb.buffers[mb.rbuf]) {
		mb.rbuf++
		mb.rp = 0
	}

	if mb.rbuf == len(mb.buffers) {
		err = io.EOF
		return
	} else if mb.rbuf > len(mb.buffers) {
		panic("mb.cur > len(mb.buffers)")
	}

	n = copy(p, mb.buffers[mb.rbuf][mb.rp:])
	mb.rp += n

	return
}

func (mb *MBuf) Full() bool {
	return mb.buffers == nil || (mb.wp == cap(mb.buffers[mb.wbuf]) && mb.wbuf+1 == len(mb.buffers))
}

func (mb *MBuf) Write(p []byte) (n int, err error) {
	b := mb.buffers[mb.wbuf]

	if mb.wp == cap(b) {
		if mb.wbuf+1 == len(mb.buffers) {
			return
		}
		mb.wbuf++
		b = mb.buffers[mb.wbuf]
		mb.wp = 0
	} else if mb.wp > cap(b) {
		panic("mb.wp > cap(b)")
	}

	n = copy(b[mb.wp:cap(b)], p)
	mb.wp += n
	// resize the buffer to account for what we just read
	mb.buffers[mb.wbuf] = mb.buffers[mb.wbuf][:mb.wp]

	return
}

func (mb *MBuf) WriteFrom(r io.Reader) (n int, err error) {
	b := mb.buffers[mb.wbuf]

	if mb.wp == cap(b) {
		if mb.wbuf+1 == len(mb.buffers) {
			return
		}
		mb.wbuf++
		b = mb.buffers[mb.wbuf]
		mb.wp = 0
	} else if mb.wp > cap(b) {
		panic("mb.wp > cap(b)")
	}

	n, err = r.Read(b[mb.wp:cap(b)])
	mb.wp += n
	// resize the buffer to account for what we just read
	mb.buffers[mb.wbuf] = mb.buffers[mb.wbuf][:mb.wp]

	return
}

func (mb *MBuf) Free() {
	for _, b := range mb.buffers {
		mb.pool.Free(b)
	}

	mb.buffers = nil
}

var bufferLog = GetLogger("buffer")

type Buffer struct {
	mu   sync.Mutex
	cond *sync.Cond

	buf    *MBuf
	reader io.ReadCloser
	err    error
}

type ReaderProvider func() (io.ReadCloser, error)

func (b Buffer) Init(buf *MBuf, r ReaderProvider) *Buffer {
	b.buf = buf
	b.cond = sync.NewCond(&b.mu)

	go func() {
		b.readLoop(r)
	}()

	return &b
}

func (b *Buffer) readLoop(r ReaderProvider) {
	for {
		b.mu.Lock()
		if b.reader == nil {
			b.reader, b.err = r()
			b.cond.Broadcast()
			if b.err != nil {
				b.mu.Unlock()
				break
			}
		}

		if b.buf == nil {
			// buffer was drained
			b.mu.Unlock()
			break
		}

		nread, err := b.buf.WriteFrom(b.reader)
		if err != nil {
			b.err = err
			b.mu.Unlock()
			break
		}
		bufferLog.Debugf("wrote %v into buffer", nread)

		if nread == 0 {
			b.reader.Close()
			b.mu.Unlock()
			break
		}

		b.mu.Unlock()
		// if we get here we've read _something_, bounce this goroutine
		// to allow another one to read
		runtime.Gosched()
	}
	bufferLog.Debugf("<-- readLoop()")
}

func (b *Buffer) readFromStream(p []byte) (n int, err error) {
	bufferLog.Debugf("reading %v from stream", len(p))

	n, err = b.reader.Read(p)
	if n != 0 && err == io.ErrUnexpectedEOF {
		err = nil
	} else {
		bufferLog.Debugf("read %v from stream", n)
	}
	return
}

func (b *Buffer) Read(p []byte) (n int, err error) {
	bufferLog.Debugf("Buffer.Read(%v)", len(p))

	b.mu.Lock()
	defer b.mu.Unlock()

	for b.reader == nil && b.err == nil {
		bufferLog.Debugf("waiting for stream")
		b.cond.Wait()
		if b.err != nil {
			err = b.err
			return
		}
	}

	// we could have received the err before Read was called
	if b.reader == nil {
		if b.err == nil {
			panic("reader and err are both nil")
		}
		err = b.err
		return
	}

	if b.buf != nil {
		bufferLog.Debugf("reading %v from buffer", len(p))

		n, err = b.buf.Read(p)
		if n == 0 {
			b.buf.Free()
			b.buf = nil
			bufferLog.Debugf("drained buffer")
			n, err = b.readFromStream(p)
		} else {
			bufferLog.Debugf("read %v from buffer", n)
		}
	} else if b.err != nil {
		err = b.err
	} else {
		n, err = b.readFromStream(p)
	}

	return
}

func (b *Buffer) Close() (err error) {
	b.mu.Lock()
	defer b.mu.Unlock()

	if b.reader != nil {
		err = b.reader.Close()
	}

	if b.buf != nil {
		b.buf.Free()
		b.buf = nil
	}

	return
}
<|endoftext|>


<commit_before>package table

import (
	"os"
	"runtime"
	"sync"

	"github.com/kezhuw/leveldb/internal/file"
	"github.com/kezhuw/leveldb/internal/files"
	"github.com/kezhuw/leveldb/internal/iterator"
	"github.com/kezhuw/leveldb/internal/keys"
	"github.com/kezhuw/leveldb/internal/options"
)

type node struct {
	f    uint64
	t    *Table
	err  error
	wg   sync.WaitGroup
	next *node
	prev *node
}

type nodeList struct {
	len  int
	root node
}

func (l *nodeList) init() {
	l.len = 0
	l.root.next = &l.root
	l.root.prev = &l.root
}

func (l *nodeList) insertAfter(n *node, at *node) {
	n.next = at.next
	at.next.prev = n
	at.next = n
	n.prev = at
}

func (l *nodeList) PushBack(n *node) {
	l.insertAfter(n, l.root.prev)
	l.len++
}

func (l *nodeList) unlink(n *node) {
	n.next.prev = n.prev
	n.prev.next = n.next
}

func (l *nodeList) Remove(n *node) *node {
	l.unlink(n)
	l.len--
	return n
}

func (l *nodeList) RemoveFront() *node {
	n := l.root.next
	l.unlink(n)
	l.len--
	return n
}

func (l *nodeList) Len() int {
	return l.len
}

func (l *nodeList) MoveToBack(n *node) {
	l.unlink(n)
	l.insertAfter(n, l.root.prev)
}

type cachePool struct {
	mu     sync.Mutex
	tables map[uint64]*node

	cap     int
	nodes   nodeList
	evicts  chan *node
	updates chan *node
}

func newCachePool(cap int) *cachePool {
	p := &cachePool{
		tables:  make(map[uint64]*node),
		cap:     cap,
		evicts:  make(chan *node, 16),
		updates: make(chan *node, 512),
	}
	go p.collect()
	return p
}

func (p *cachePool) Close() {
	close(p.updates)
}

var deletedNode = new(node)

func (p *cachePool) touchNode(n *node) {
	select {
	case p.updates <- n:
	default:
	}
}

func (p *cachePool) updateNode(n *node) {
	switch n.next {
	case nil:
		p.nodes.PushBack(n)
		if p.nodes.Len() > p.cap {
			n := p.nodes.RemoveFront()
			p.mu.Lock()
			delete(p.tables, n.f)
			p.mu.Unlock()
			n.next = deletedNode
			n.prev = nil
		}
	case deletedNode:
	default:
		p.nodes.MoveToBack(n)
	}
}

func (p *cachePool) removeNode(n *node) {
	switch n.next {
	case nil:
		n.next = deletedNode
	case deletedNode:
	default:
		p.nodes.Remove(n)
		n.next = deletedNode
		n.prev = nil
	}
}

func (p *cachePool) collect() {
	p.nodes.init()
	evicts := p.evicts
	updates := p.updates
	for {
		select {
		case n, ok := <-updates:
			if !ok {
				return
			}
			p.updateNode(n)
		case n := <-evicts:
			p.removeNode(n)
		}
	}
}

type Cache struct {
	dbname  string
	fs      file.FileSystem
	blocks  *BlockCache
	options *options.Options
	*cachePool
}

func (c *Cache) load(fileNumber, fileSize uint64, n *node) (*Table, error) {
	defer n.wg.Done()
	tableName := files.TableFileName(c.dbname, fileNumber)
	tableFile, err := c.fs.Open(tableName, os.O_RDONLY)
	if err != nil {
		var err1 error
		tableName = files.SSTTableFileName(c.dbname, fileNumber)
		tableFile, err1 = c.fs.Open(tableName, os.O_RDONLY)
		if err1 != nil {
			n.t = nil
			n.err = err
			return nil, err
		}
	}
	n.t, n.err = OpenTable(tableFile, c.blocks, c.options, fileNumber, fileSize)
	return n.t, n.err
}

func (c *Cache) open(fileNumber, fileSize uint64) (*Table, error) {
	c.mu.Lock()
	n := c.tables[fileNumber]
	if n != nil {
		c.mu.Unlock()
		n.wg.Wait()
		c.touchNode(n)
		return n.t, n.err
	}
	n = &node{f: fileNumber}
	c.tables[fileNumber] = n
	n.wg.Add(1)
	c.mu.Unlock()
	c.touchNode(n)
	return c.load(fileNumber, fileSize, n)
}

func (c *Cache) Evict(fileNumber uint64) {
	c.mu.Lock()
	n := c.tables[fileNumber]
	delete(c.tables, fileNumber)
	c.mu.Unlock()
	if n != nil {
		c.evicts <- n
	}
}

func (c *Cache) Get(fileNumber uint64, fileSize uint64, ikey keys.InternalKey, opts *options.ReadOptions) ([]byte, error, bool) {
	t, err := c.open(fileNumber, fileSize)
	if err != nil {
		return nil, err, true
	}
	return t.Get(ikey, opts)
}

func (c *Cache) NewIterator(fileNumber uint64, fileSize uint64, opts *options.ReadOptions) iterator.Iterator {
	t, err := c.open(fileNumber, fileSize)
	if err != nil {
		return iterator.Error(err)
	}
	return t.NewIterator(opts)
}

func (c *Cache) finalize() {
	c.cachePool.Close()
}

func NewCache(dbname string, opts *options.Options) *Cache {
	c := &Cache{
		dbname:    dbname,
		fs:        opts.FileSystem,
		options:   opts,
		cachePool: newCachePool(opts.MaxOpenFiles),
		blocks:    NewBlockCache(opts.BlockCacheCapacity),
	}
	runtime.SetFinalizer(c, (*Cache).finalize)
	return c
}
<commit_msg>Try .sst only if .ldb not exist<commit_after>package table

import (
	"os"
	"runtime"
	"sync"

	"github.com/kezhuw/leveldb/internal/file"
	"github.com/kezhuw/leveldb/internal/files"
	"github.com/kezhuw/leveldb/internal/iterator"
	"github.com/kezhuw/leveldb/internal/keys"
	"github.com/kezhuw/leveldb/internal/options"
)

type node struct {
	f    uint64
	t    *Table
	err  error
	wg   sync.WaitGroup
	next *node
	prev *node
}

type nodeList struct {
	len  int
	root node
}

func (l *nodeList) init() {
	l.len = 0
	l.root.next = &l.root
	l.root.prev = &l.root
}

func (l *nodeList) insertAfter(n *node, at *node) {
	n.next = at.next
	at.next.prev = n
	at.next = n
	n.prev = at
}

func (l *nodeList) PushBack(n *node) {
	l.insertAfter(n, l.root.prev)
	l.len++
}

func (l *nodeList) unlink(n *node) {
	n.next.prev = n.prev
	n.prev.next = n.next
}

func (l *nodeList) Remove(n *node) *node {
	l.unlink(n)
	l.len--
	return n
}

func (l *nodeList) RemoveFront() *node {
	n := l.root.next
	l.unlink(n)
	l.len--
	return n
}

func (l *nodeList) Len() int {
	return l.len
}

func (l *nodeList) MoveToBack(n *node) {
	l.unlink(n)
	l.insertAfter(n, l.root.prev)
}

type cachePool struct {
	mu     sync.Mutex
	tables map[uint64]*node

	cap     int
	nodes   nodeList
	evicts  chan *node
	updates chan *node
}

func newCachePool(cap int) *cachePool {
	p := &cachePool{
		tables:  make(map[uint64]*node),
		cap:     cap,
		evicts:  make(chan *node, 16),
		updates: make(chan *node, 512),
	}
	go p.collect()
	return p
}

func (p *cachePool) Close() {
	close(p.updates)
}

var deletedNode = new(node)

func (p *cachePool) touchNode(n *node) {
	select {
	case p.updates <- n:
	default:
	}
}

func (p *cachePool) updateNode(n *node) {
	switch n.next {
	case nil:
		p.nodes.PushBack(n)
		if p.nodes.Len() > p.cap {
			n := p.nodes.RemoveFront()
			p.mu.Lock()
			delete(p.tables, n.f)
			p.mu.Unlock()
			n.next = deletedNode
			n.prev = nil
		}
	case deletedNode:
	default:
		p.nodes.MoveToBack(n)
	}
}

func (p *cachePool) removeNode(n *node) {
	switch n.next {
	case nil:
		n.next = deletedNode
	case deletedNode:
	default:
		p.nodes.Remove(n)
		n.next = deletedNode
		n.prev = nil
	}
}

func (p *cachePool) collect() {
	p.nodes.init()
	evicts := p.evicts
	updates := p.updates
	for {
		select {
		case n, ok := <-updates:
			if !ok {
				return
			}
			p.updateNode(n)
		case n := <-evicts:
			p.removeNode(n)
		}
	}
}

type Cache struct {
	dbname  string
	fs      file.FileSystem
	blocks  *BlockCache
	options *options.Options
	*cachePool
}

func (c *Cache) load(fileNumber, fileSize uint64, n *node) (*Table, error) {
	defer n.wg.Done()
	tableName := files.TableFileName(c.dbname, fileNumber)
	tableFile, err := c.fs.Open(tableName, os.O_RDONLY)
	if os.IsNotExist(err) {
		tableName = files.SSTTableFileName(c.dbname, fileNumber)
		tableFile, err = c.fs.Open(tableName, os.O_RDONLY)
	}
	if err != nil {
		n.t = nil
		n.err = err
		return nil, err
	}
	n.t, n.err = OpenTable(tableFile, c.blocks, c.options, fileNumber, fileSize)
	return n.t, n.err
}

func (c *Cache) open(fileNumber, fileSize uint64) (*Table, error) {
	c.mu.Lock()
	n := c.tables[fileNumber]
	if n != nil {
		c.mu.Unlock()
		n.wg.Wait()
		c.touchNode(n)
		return n.t, n.err
	}
	n = &node{f: fileNumber}
	c.tables[fileNumber] = n
	n.wg.Add(1)
	c.mu.Unlock()
	c.touchNode(n)
	return c.load(fileNumber, fileSize, n)
}

func (c *Cache) Evict(fileNumber uint64) {
	c.mu.Lock()
	n := c.tables[fileNumber]
	delete(c.tables, fileNumber)
	c.mu.Unlock()
	if n != nil {
		c.evicts <- n
	}
}

func (c *Cache) Get(fileNumber uint64, fileSize uint64, ikey keys.InternalKey, opts *options.ReadOptions) ([]byte, error, bool) {
	t, err := c.open(fileNumber, fileSize)
	if err != nil {
		return nil, err, true
	}
	return t.Get(ikey, opts)
}

func (c *Cache) NewIterator(fileNumber uint64, fileSize uint64, opts *options.ReadOptions) iterator.Iterator {
	t, err := c.open(fileNumber, fileSize)
	if err != nil {
		return iterator.Error(err)
	}
	return t.NewIterator(opts)
}

func (c *Cache) finalize() {
	c.cachePool.Close()
}

func NewCache(dbname string, opts *options.Options) *Cache {
	c := &Cache{
		dbname:    dbname,
		fs:        opts.FileSystem,
		options:   opts,
		cachePool: newCachePool(opts.MaxOpenFiles),
		blocks:    NewBlockCache(opts.BlockCacheCapacity),
	}
	runtime.SetFinalizer(c, (*Cache).finalize)
	return c
}
<|endoftext|>


<commit_before>package web

import (
	"html/template"
	"strings"

	"github.com/mdlayher/mdlayher.com/internal/github"
	"github.com/mdlayher/mdlayher.com/internal/medium"
)

// Content is the top-level object for the HTML template.
type Content struct {
	Static StaticContent
	GitHub GitHubContent
	Medium MediumContent
}

// StaticContent contains statically defined content for the HTML template.
type StaticContent struct {
	Domain  string
	Name    string
	Tagline string
	Links   []Link
}

// GitHubContent contains dynamic content from GitHub for the HTML template.
type GitHubContent struct {
	Repositories []*github.Repository
}

// MediumContent contains dynamic content from Medium for the HTML template.
type MediumContent struct {
	Posts []*medium.Post
}

// A Link is a hyperlink and a display title for that link.
type Link struct {
	Title string
	Link  string
}

// tmpl is the HTML template served to users of the site.
var tmpl = template.Must(template.New("html").Parse(strings.TrimSpace(`
<!DOCTYPE html>

	<title>{{.Static.Name}}</title>
	<meta charset='utf-8' />
	<meta name="description" content="{{.Static.Name}} - {{.Static.Domain}}" />

{{.Static.Name}}

{{.Static.Tagline}}

		{{range .Static.Links}}
{{.Title}}

		{{end}}

	{{if .GitHub.Repositories}}

Open Source

		{{range .GitHub.Repositories}}
{{.Name}}

{{.Description}}

		{{end}}

	{{end}}
	{{if .Medium.Posts}}

Blog Posts

	{{range .Medium.Posts}}
{{.Title}}

{{.Subtitle}}

	{{end}}

	{{end}}

`)))
<commit_msg>internal/web: use sans-serif font<commit_after>package web

import (
	"html/template"
	"strings"

	"github.com/mdlayher/mdlayher.com/internal/github"
	"github.com/mdlayher/mdlayher.com/internal/medium"
)

// Content is the top-level object for the HTML template.
type Content struct {
	Static StaticContent
	GitHub GitHubContent
	Medium MediumContent
}

// StaticContent contains statically defined content for the HTML template.
type StaticContent struct {
	Domain  string
	Name    string
	Tagline string
	Links   []Link
}

// GitHubContent contains dynamic content from GitHub for the HTML template.
type GitHubContent struct {
	Repositories []*github.Repository
}

// MediumContent contains dynamic content from Medium for the HTML template.
type MediumContent struct {
	Posts []*medium.Post
}

// A Link is a hyperlink and a display title for that link.
type Link struct {
	Title string
	Link  string
}

// tmpl is the HTML template served to users of the site.
var tmpl = template.Must(template.New("html").Parse(strings.TrimSpace(`
<!DOCTYPE html>

	<title>{{.Static.Name}}</title>
	<meta charset="utf-8" />
	<meta name="description" content="{{.Static.Name}} - {{.Static.Domain}}" />

{{.Static.Name}}

{{.Static.Tagline}}

		{{range .Static.Links}}
{{.Title}}

		{{end}}

	{{if .GitHub.Repositories}}

Open Source

		{{range .GitHub.Repositories}}
{{.Name}}

{{.Description}}

		{{end}}

	{{end}}
	{{if .Medium.Posts}}

Blog Posts

	{{range .Medium.Posts}}
{{.Title}}

{{.Subtitle}}

	{{end}}

	{{end}}

`)))
<|endoftext|>


<commit_before>// Copyright 2014, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package ioutil

import "io"
import "os"
import "errors"

type MemoryReader struct {
	rd    io.Reader
	buf   []byte
	rdPtr int64
	wrPtr int64
}

func NewMemoryReader(rd io.Reader, buf []byte) *MemoryReader {
	return &MemoryReader{rd, buf, 0, 0}
}

func (r *MemoryReader) Read(data []byte) (cnt int, err error) {
	// If there is data to read from ring buffer, use it
	for len(data) > 0 && r.wrPtr > r.rdPtr {
		bufLen := int64(len(r.buf))
		offLo, offHi := r.rdPtr%bufLen, r.wrPtr%bufLen
		if offHi <= offLo {
			offHi = bufLen
		}
		cpyCnt := copy(data, r.buf[offLo:offHi])
		data = data[cpyCnt:]
		cnt += cpyCnt
		r.rdPtr += int64(cpyCnt)
	}

	// Top off the input buffer with actual Read() call
	var rdCnt int
	rdCnt, err = r.rd.Read(data)
	cnt += rdCnt

	// Write data to ring buffer
	if len(data) > len(r.buf) {
		skipCnt := len(data) - len(r.buf)
		data = data[skipCnt:]
		r.rdPtr += int64(skipCnt)
		r.wrPtr += int64(skipCnt)
	}
	for len(data) > 0 {
		off := r.wrPtr % int64(len(r.buf))
		cpyCnt := copy(r.buf[off:], data)
		data = data[cpyCnt:]
		r.rdPtr += int64(cpyCnt)
		r.wrPtr += int64(cpyCnt)
	}

	return cnt, err
}

func (r *MemoryReader) Seek(offset int64, whence int) (pos int64, err error) {
	switch whence {
	case os.SEEK_SET:
		pos = offset
	case os.SEEK_CUR:
		pos = offset + r.rdPtr
	default:
		return 0, errors.New("ioutil.MemoryReader.Seek: invalid whence")
	}
	if posLo, _, posHi := r.SeekOffsets(); pos < posLo || pos > posHi {
		return 0, errors.New("ioutil.MemoryReader.Seek: buffer out of bounds")
	}
	r.rdPtr = pos
	return
}

func (r *MemoryReader) SeekOffsets() (int64, int64, int64) {
	ptrLo := r.wrPtr - int64(len(r.buf))
	if ptrLo < 0 {
		ptrLo = 0
	}
	return ptrLo, r.rdPtr, r.wrPtr
}
<commit_msg>Fixed MemoryReader bug with updating data slice<commit_after>// Copyright 2014, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package ioutil

import "io"
import "os"
import "errors"

type MemoryReader struct {
	rd    io.Reader
	buf   []byte
	rdPtr int64
	wrPtr int64
}

func NewMemoryReader(rd io.Reader, buf []byte) *MemoryReader {
	return &MemoryReader{rd, buf, 0, 0}
}

func (r *MemoryReader) Read(data []byte) (cnt int, err error) {
	// If there is data to read from ring buffer, use it
	for len(data) > 0 && r.wrPtr > r.rdPtr {
		bufLen := int64(len(r.buf))
		offLo, offHi := r.rdPtr%bufLen, r.wrPtr%bufLen
		if offHi <= offLo {
			offHi = bufLen
		}
		cpyCnt := copy(data, r.buf[offLo:offHi])
		data = data[cpyCnt:]
		cnt += cpyCnt
		r.rdPtr += int64(cpyCnt)
	}

	// Top off the input buffer with actual Read() call
	var rdCnt int
	rdCnt, err = r.rd.Read(data)
	data = data[:rdCnt]
	cnt += rdCnt

	// Write data to ring buffer
	if len(data) > len(r.buf) {
		skipCnt := len(data) - len(r.buf)
		data = data[skipCnt:]
		r.rdPtr += int64(skipCnt)
		r.wrPtr += int64(skipCnt)
	}
	for len(data) > 0 {
		off := r.wrPtr % int64(len(r.buf))
		cpyCnt := copy(r.buf[off:], data)
		data = data[cpyCnt:]
		r.rdPtr += int64(cpyCnt)
		r.wrPtr += int64(cpyCnt)
	}

	return cnt, err
}

func (r *MemoryReader) Seek(offset int64, whence int) (pos int64, err error) {
	switch whence {
	case os.SEEK_SET:
		pos = offset
	case os.SEEK_CUR:
		pos = offset + r.rdPtr
	default:
		return 0, errors.New("ioutil.MemoryReader.Seek: invalid whence")
	}
	if posLo, _, posHi := r.SeekOffsets(); pos < posLo || pos > posHi {
		return 0, errors.New("ioutil.MemoryReader.Seek: buffer out of bounds")
	}
	r.rdPtr = pos
	return
}

func (r *MemoryReader) SeekOffsets() (int64, int64, int64) {
	ptrLo := r.wrPtr - int64(len(r.buf))
	if ptrLo < 0 {
		ptrLo = 0
	}
	return ptrLo, r.rdPtr, r.wrPtr
}
<|endoftext|>


<commit_before>package formatter

import (
	"bytes"
	"errors"
	"io"

	"github.com/nwidger/jsoncolor"
	"github.com/tidwall/gjson"
)

type jsonFormatter struct {
	parsedBody gjson.Result
	TextFormatter
}

func (f *jsonFormatter) Format(writer io.Writer, data []byte) error {
	jsonFormatter := jsoncolor.NewFormatter()
	buf := bytes.NewBuffer(make([]byte, 0, len(data)))
	err := jsonFormatter.Format(buf, data)
	if err == nil {
		writer.Write(buf.Bytes())
		return nil
	}
	return errors.New("json formatter error")
}

func (f *jsonFormatter) Title() string {
	return "[json]"
}

func (f *jsonFormatter) Search(q string, body []byte) ([]string, error) {
	if q != "" {
		if f.parsedBody.Type != gjson.JSON {
			f.parsedBody = gjson.ParseBytes(body)
		}
		searchResult := f.parsedBody.Get(q)
		if searchResult.Type == gjson.Null {
			return nil, errors.New("Invalid gjson query or no results found")
		}
		if searchResult.Type != gjson.JSON {
			return []string{searchResult.String()}, nil
		}
		body = []byte(searchResult.String())
	}
	jsonFormatter := jsoncolor.NewFormatter()
	buf := bytes.NewBuffer(make([]byte, 0, len(body)))
	err := jsonFormatter.Format(buf, body)
	if err != nil {
		return nil, errors.New("Invalid results")
	}
	return []string{string(buf.Bytes())}, nil
}
<commit_msg>[fix] update json formatting to indent the formatted output - closes #130<commit_after>package formatter

import (
	"bytes"
	"errors"
	"io"

	"github.com/nwidger/jsoncolor"
	"github.com/tidwall/gjson"
)

type jsonFormatter struct {
	parsedBody gjson.Result
	TextFormatter
}

func (f *jsonFormatter) Format(writer io.Writer, data []byte) error {
	jsonFormatter := jsoncolor.NewFormatter()
	jsonFormatter.Indent = "  "
	buf := bytes.NewBuffer(make([]byte, 0, len(data)))
	err := jsonFormatter.Format(buf, data)
	if err == nil {
		writer.Write(buf.Bytes())
		return nil
	}
	return errors.New("json formatter error")
}

func (f *jsonFormatter) Title() string {
	return "[json]"
}

func (f *jsonFormatter) Search(q string, body []byte) ([]string, error) {
	if q != "" {
		if f.parsedBody.Type != gjson.JSON {
			f.parsedBody = gjson.ParseBytes(body)
		}
		searchResult := f.parsedBody.Get(q)
		if searchResult.Type == gjson.Null {
			return nil, errors.New("Invalid gjson query or no results found")
		}
		if searchResult.Type != gjson.JSON {
			return []string{searchResult.String()}, nil
		}
		body = []byte(searchResult.String())
	}
	jsonFormatter := jsoncolor.NewFormatter()
	jsonFormatter.Indent = "  "
	buf := bytes.NewBuffer(make([]byte, 0, len(body)))
	err := jsonFormatter.Format(buf, body)
	if err != nil {
		return nil, errors.New("Invalid results")
	}
	return []string{string(buf.Bytes())}, nil
}
<|endoftext|>