<commit_before>package spotify

import (
	"strings"

	"github.com/fabiofalci/sconsify/infrastructure"
	"github.com/fabiofalci/sconsify/sconsify"
	sp "github.com/op/go-libspotify/spotify"
	webspotify "github.com/zmb3/spotify"
	"strconv"
)

func (spotify *Spotify) initPlaylist() error {
	playlists := sconsify.InitPlaylists()

	allPlaylists, err := spotify.session.Playlists()
	if err != nil {
		return err
	}
	allPlaylists.Wait()
	var folderPlaylists []*sconsify.Playlist
	var folder *sp.PlaylistFolder
	infrastructure.Debugf("# of playlists %v", allPlaylists.Playlists())
	for i := 0; i < allPlaylists.Playlists(); i++ {
		if allPlaylists.PlaylistType(i) == sp.PlaylistTypeStartFolder {
			folder, _ = allPlaylists.Folder(i)
			folderPlaylists = make([]*sconsify.Playlist, 0)
			infrastructure.Debugf("Opening folder '%v' (%v)", folder.Id(), folder.Name())
		} else if allPlaylists.PlaylistType(i) == sp.PlaylistTypeEndFolder {
			if folder != nil {
				playlists.AddPlaylist(sconsify.InitFolder(strconv.Itoa(int(folder.Id())), folder.Name(), folderPlaylists))
				infrastructure.Debugf("Closing folder '%v' (%v)", folder.Id(), folder.Name())
			} else {
				infrastructure.Debug("Closing a null folder, this doesn't look right ")
			}
			folderPlaylists = nil
			folder = nil
		}

		if allPlaylists.PlaylistType(i) != sp.PlaylistTypePlaylist {
			continue
		}

		playlist := allPlaylists.Playlist(i)
		playlist.Wait()
		if spotify.canAddPlaylist(playlist, allPlaylists.PlaylistType(i)) {
			id := playlist.Link().String()
			infrastructure.Debugf("Playlist '%v' (%v)", id, playlist.Name())
			tracks := make([]*sconsify.Track, playlist.Tracks())
			infrastructure.Debugf("\t# of tracks %v", playlist.Tracks())
			for i := 0; i < playlist.Tracks(); i++ {
				tracks[i] = spotify.initTrack(playlist.Track(i))
			}
			if folderPlaylists == nil {
				playlists.AddPlaylist(sconsify.InitPlaylist(id, playlist.Name(), tracks))
			} else {
				folderPlaylists = append(folderPlaylists, sconsify.InitSubPlaylist(id, playlist.Name(), tracks))
			}
		}
	}

	webApiCache := spotify.loadWebApiCacheIfNecessary()
	if spotify.client != nil {
		playlists.AddPlaylist(sconsify.InitOnDemandFolder("Albums", "*Albums", true, func(playlist *sconsify.Playlist) {
			spotify.loadAlbums(playlist, webApiCache)
			spotify.persistWebApiCache(webApiCache)
		}))
		playlists.AddPlaylist(sconsify.InitOnDemandPlaylist("Songs", "*Songs", false, func(playlist *sconsify.Playlist) {
			spotify.loadSongs(playlist, webApiCache)
			spotify.persistWebApiCache(webApiCache)
		}))
		playlists.AddPlaylist(sconsify.InitOnDemandFolder("New Releases", "*New Releases", true, func(playlist *sconsify.Playlist) {
			spotify.loadNewReleases(playlist, webApiCache)
			spotify.persistWebApiCache(webApiCache)
		}))
	} else {
		if webApiCache.Albums != nil {
			playlist := sconsify.InitOnDemandFolder("Albums", "*Albums", true, func(playlist *sconsify.Playlist) {
				spotify.loadAlbums(playlist, webApiCache)
			})
			playlist.ExecuteLoad()
			playlists.AddPlaylist(playlist)
		}
		if webApiCache.Songs != nil {
			playlist := sconsify.InitOnDemandPlaylist("Songs", "*Songs", true, func(playlist *sconsify.Playlist) {
				spotify.loadSongs(playlist, webApiCache)
			})
			playlist.ExecuteLoad()
			playlists.AddPlaylist(playlist)
		}
		if webApiCache.NewReleases != nil {
			playlist := sconsify.InitOnDemandFolder("New Releases", "*New Releases", true, func(playlist *sconsify.Playlist) {
				spotify.loadNewReleases(playlist, webApiCache)
			})
			playlist.ExecuteLoad()
			playlists.AddPlaylist(playlist)
		}
	}

	spotify.events.NewPlaylist(playlists)
	return nil
}

func (spotify *Spotify) loadWebApiCacheIfNecessary() *WebApiCache {
	if spotify.client != nil {
		return &WebApiCache{}
	}
	return spotify.loadWebApiCache()
}

func (spotify *Spotify) loadAlbums(playlist *sconsify.Playlist, webApiCache *WebApiCache) {
	if spotify.client != nil {
		if savedAlbumPage, err := spotify.client.CurrentUsersAlbumsOpt(createWebSpotifyOptions(50, playlist.Playlists())); err == nil {
			webApiCache.Albums = savedAlbumPage.Albums
		}
	}

	if webApiCache.Albums != nil {
		for _, album := range webApiCache.Albums {
			tracks := make([]*sconsify.Track, len(album.Tracks.Tracks))
			for j, track := range album.Tracks.Tracks {
				webArtist := track.Artists[0]
				artist := sconsify.InitArtist(string(webArtist.URI), webArtist.Name)
				tracks[j] = sconsify.InitWebApiTrack(string(track.URI), artist, track.Name, track.TimeDuration().String())
			}
			playlist.AddPlaylist(sconsify.InitSubPlaylist(string(album.URI), album.Name, tracks))
		}
		playlist.OpenFolder()
	}

}

func (spotify *Spotify) loadSongs(playlist *sconsify.Playlist, webApiCache *WebApiCache) {
	var partialSongs []webspotify.SavedTrack
	if spotify.client != nil {
		if savedTrackPage, err := spotify.client.CurrentUsersTracksOpt(createWebSpotifyOptions(50, playlist.Tracks())); err == nil {
			partialSongs = savedTrackPage.Tracks
			if webApiCache.Songs == nil {
				webApiCache.Songs = make([]webspotify.SavedTrack, 0)
			}
			webApiCache.Songs = append(webApiCache.Songs, partialSongs...)
		}
	}

	if webApiCache.Songs != nil {
		if partialSongs == nil {
			partialSongs = webApiCache.Songs
		}
		for i, track := range partialSongs {
			webApiCache.Songs[i] = track
			webArtist := track.Artists[0]
			artist := sconsify.InitArtist(string(webArtist.URI), webArtist.Name)
			playlist.AddTrack(sconsify.InitWebApiTrack(string(track.URI), artist, track.Name, track.TimeDuration().String()))
		}
	}
}

func (spotify *Spotify) loadNewReleases(playlist *sconsify.Playlist, webApiCache *WebApiCache) {
	if spotify.client != nil {
		if _, simplePlaylistPage, err := spotify.client.FeaturedPlaylistsOpt(&webspotify.PlaylistOptions{Options: *createWebSpotifyOptions(50, playlist.Playlists())}); err == nil {
			webApiCache.NewReleases = make([]webspotify.FullPlaylist, len(simplePlaylistPage.Playlists))
			for i, webPlaylist := range simplePlaylistPage.Playlists {
				if fullPlaylist, err := spotify.client.GetPlaylist(webPlaylist.Owner.ID, webPlaylist.ID); err == nil {
					webApiCache.NewReleases[i] = *fullPlaylist
				}
			}
		}
	}

	if webApiCache.NewReleases != nil {
		for _, fullPlaylist := range webApiCache.NewReleases {
			tracks := make([]*sconsify.Track, len(fullPlaylist.Tracks.Tracks))
			for i, track := range fullPlaylist.Tracks.Tracks {
				webArtist := track.Track.Artists[0]
				artist := sconsify.InitArtist(string(webArtist.URI), webArtist.Name)
				tracks[i] = sconsify.InitWebApiTrack(string(track.Track.URI), artist, track.Track.Name, track.Track.TimeDuration().String())
			}
			playlist.AddPlaylist(sconsify.InitSubPlaylist(string(fullPlaylist.URI), fullPlaylist.Name, tracks))
		}
		playlist.OpenFolder()
	}
}

func createWebSpotifyOptions(limit int, offset int) *webspotify.Options {
	return &webspotify.Options{Limit: &limit, Offset: &offset}
}

func (spotify *Spotify) initTrack(playlistTrack *sp.PlaylistTrack) *sconsify.Track {
	track := playlistTrack.Track()
	track.Wait()
	for i := 0; i < track.Artists(); i++ {
		track.Artist(i).Wait()
	}
	infrastructure.Debugf("\tTrack '%v' (%v)", track.Link().String(), track.Name())
	return sconsify.ToSconsifyTrack(track)
}

func (spotify *Spotify) canAddPlaylist(playlist *sp.Playlist, playlistType sp.PlaylistType) bool {
	return playlistType == sp.PlaylistTypePlaylist && spotify.isOnFilter(playlist.Name())
}

func (spotify *Spotify) isOnFilter(playlist string) bool {
	if spotify.playlistFilter == nil {
		return true
	}
	for _, filter := range spotify.playlistFilter {
		if filter == playlist {
			return true
		}
	}
	return false
}

func (spotify *Spotify) setPlaylistFilter(playlistFilter string) {
	if playlistFilter == "" {
		return
	}
	spotify.playlistFilter = strings.Split(playlistFilter, ",")
	for i := range spotify.playlistFilter {
		spotify.playlistFilter[i] = strings.Trim(spotify.playlistFilter[i], " ")
	}
}
<commit_msg>If using -web-api flag, load playlist from web api<commit_after>package spotify

import (
	"strings"

	"github.com/fabiofalci/sconsify/infrastructure"
	"github.com/fabiofalci/sconsify/sconsify"
	sp "github.com/op/go-libspotify/spotify"
	webspotify "github.com/zmb3/spotify"
	"strconv"
	"fmt"
)

func (spotify *Spotify) initPlaylist() error {
	playlists := sconsify.InitPlaylists()

	if spotify.client != nil {
		if privateUser, err := spotify.client.CurrentUser(); err == nil {
			offset := 0
			total := 1
			for offset <= total {
				offset, total = spotify.loadPlaylists(offset, privateUser, playlists)
				if offset > total {
					fmt.Printf("Loaded %v from %v playlists\n", total, total)
				} else {
					fmt.Printf("Loaded %v from %v playlists\n", offset, total)
				}
			}
		} else {
			panic(err)
		}
	} else {
		fmt.Print("Warning: not using -web-api flag. Sconsify will load playlists using deprecated libspotify API. If not working try -web-api flag.")
		allPlaylists, err := spotify.session.Playlists()
		if err != nil {
			return err
		}
		allPlaylists.Wait()
		var folderPlaylists []*sconsify.Playlist
		var folder *sp.PlaylistFolder
		infrastructure.Debugf("# of playlists %v", allPlaylists.Playlists())
		for i := 0; i < allPlaylists.Playlists(); i++ {
			if allPlaylists.PlaylistType(i) == sp.PlaylistTypeStartFolder {
				folder, _ = allPlaylists.Folder(i)
				folderPlaylists = make([]*sconsify.Playlist, 0)
				infrastructure.Debugf("Opening folder '%v' (%v)", folder.Id(), folder.Name())
			} else if allPlaylists.PlaylistType(i) == sp.PlaylistTypeEndFolder {
				if folder != nil {
					playlists.AddPlaylist(sconsify.InitFolder(strconv.Itoa(int(folder.Id())), folder.Name(), folderPlaylists))
					infrastructure.Debugf("Closing folder '%v' (%v)", folder.Id(), folder.Name())
				} else {
					infrastructure.Debug("Closing a null folder, this doesn't look right ")
				}
				folderPlaylists = nil
				folder = nil
			}

			if allPlaylists.PlaylistType(i) != sp.PlaylistTypePlaylist {
				continue
			}

			playlist := allPlaylists.Playlist(i)
			playlist.Wait()
			if spotify.canAddPlaylist(playlist, allPlaylists.PlaylistType(i)) {
				id := playlist.Link().String()
				infrastructure.Debugf("Playlist '%v' (%v)", id, playlist.Name())
				tracks := make([]*sconsify.Track, playlist.Tracks())
				infrastructure.Debugf("\t# of tracks %v", playlist.Tracks())
				for i := 0; i < playlist.Tracks(); i++ {
					tracks[i] = spotify.initTrack(playlist.Track(i))
				}
				if folderPlaylists == nil {
					playlists.AddPlaylist(sconsify.InitPlaylist(id, playlist.Name(), tracks))
				} else {
					folderPlaylists = append(folderPlaylists, sconsify.InitSubPlaylist(id, playlist.Name(), tracks))
				}
			}
		}
	}

	webApiCache := spotify.loadWebApiCacheIfNecessary()
	if spotify.client != nil {
		playlists.AddPlaylist(sconsify.InitOnDemandFolder("Albums", "*Albums", true, func(playlist *sconsify.Playlist) {
			spotify.loadAlbums(playlist, webApiCache)
			spotify.persistWebApiCache(webApiCache)
		}))
		playlists.AddPlaylist(sconsify.InitOnDemandPlaylist("Songs", "*Songs", false, func(playlist *sconsify.Playlist) {
			spotify.loadSongs(playlist, webApiCache)
			spotify.persistWebApiCache(webApiCache)
		}))
		playlists.AddPlaylist(sconsify.InitOnDemandFolder("New Releases", "*New Releases", true, func(playlist *sconsify.Playlist) {
			spotify.loadNewReleases(playlist, webApiCache)
			spotify.persistWebApiCache(webApiCache)
		}))
	} else {
		if webApiCache.Albums != nil {
			playlist := sconsify.InitOnDemandFolder("Albums", "*Albums", true, func(playlist *sconsify.Playlist) {
				spotify.loadAlbums(playlist, webApiCache)
			})
			playlist.ExecuteLoad()
			playlists.AddPlaylist(playlist)
		}
		if webApiCache.Songs != nil {
			playlist := sconsify.InitOnDemandPlaylist("Songs", "*Songs", true, func(playlist *sconsify.Playlist) {
				spotify.loadSongs(playlist, webApiCache)
			})
			playlist.ExecuteLoad()
			playlists.AddPlaylist(playlist)
		}
		if webApiCache.NewReleases != nil {
			playlist := sconsify.InitOnDemandFolder("New Releases", "*New Releases", true, func(playlist *sconsify.Playlist) {
				spotify.loadNewReleases(playlist, webApiCache)
			})
			playlist.ExecuteLoad()
			playlists.AddPlaylist(playlist)
		}
	}

	spotify.events.NewPlaylist(playlists)
	return nil
}

func (spotify *Spotify) loadPlaylists(offset int, privateUser *webspotify.PrivateUser, playlists *sconsify.Playlists) (int, int) {
	limit := 20
	options := &webspotify.Options{Limit: &limit, Offset: &offset}
	if simplePlaylistPage, err := spotify.client.GetPlaylistsForUserOpt(privateUser.ID, options); err == nil {
		for _, webPlaylist := range simplePlaylistPage.Playlists {
			if fullPlaylist, err := spotify.client.GetPlaylist(webPlaylist.Owner.ID, webPlaylist.ID); err == nil {
				tracks := make([]*sconsify.Track, len(fullPlaylist.Tracks.Tracks))
				for i, track := range fullPlaylist.Tracks.Tracks {
					webArtist := track.Track.Artists[0]
					artist := sconsify.InitArtist(string(webArtist.URI), webArtist.Name)
					tracks[i] = sconsify.InitWebApiTrack(string(track.Track.URI), artist, track.Track.Name, track.Track.TimeDuration().String())
				}
				playlist := sconsify.InitPlaylist(string(fullPlaylist.URI), fullPlaylist.Name, tracks)
				playlists.AddPlaylist(playlist)
			}
		}

		// simplePlaylistPage.Offset is returning 0 instead of offset
		return offset + limit, simplePlaylistPage.Total
	}

	return 0, 0
}

func (spotify *Spotify) loadWebApiCacheIfNecessary() *WebApiCache {
	if spotify.client != nil {
		return &WebApiCache{}
	}
	return spotify.loadWebApiCache()
}

func (spotify *Spotify) loadAlbums(playlist *sconsify.Playlist, webApiCache *WebApiCache) {
	if spotify.client != nil {
		if savedAlbumPage, err := spotify.client.CurrentUsersAlbumsOpt(createWebSpotifyOptions(50, playlist.Playlists())); err == nil {
			webApiCache.Albums = savedAlbumPage.Albums
		}
	}

	if webApiCache.Albums != nil {
		for _, album := range webApiCache.Albums {
			tracks := make([]*sconsify.Track, len(album.Tracks.Tracks))
			for j, track := range album.Tracks.Tracks {
				webArtist := track.Artists[0]
				artist := sconsify.InitArtist(string(webArtist.URI), webArtist.Name)
				tracks[j] = sconsify.InitWebApiTrack(string(track.URI), artist, track.Name, track.TimeDuration().String())
			}
			playlist.AddPlaylist(sconsify.InitSubPlaylist(string(album.URI), album.Name, tracks))
		}
		playlist.OpenFolder()
	}

}

func (spotify *Spotify) loadSongs(playlist *sconsify.Playlist, webApiCache *WebApiCache) {
	var partialSongs []webspotify.SavedTrack
	if spotify.client != nil {
		if savedTrackPage, err := spotify.client.CurrentUsersTracksOpt(createWebSpotifyOptions(50, playlist.Tracks())); err == nil {
			partialSongs = savedTrackPage.Tracks
			if webApiCache.Songs == nil {
				webApiCache.Songs = make([]webspotify.SavedTrack, 0)
			}
			webApiCache.Songs = append(webApiCache.Songs, partialSongs...)
		}
	}

	if webApiCache.Songs != nil {
		if partialSongs == nil {
			partialSongs = webApiCache.Songs
		}
		for i, track := range partialSongs {
			webApiCache.Songs[i] = track
			webArtist := track.Artists[0]
			artist := sconsify.InitArtist(string(webArtist.URI), webArtist.Name)
			playlist.AddTrack(sconsify.InitWebApiTrack(string(track.URI), artist, track.Name, track.TimeDuration().String()))
		}
	}
}

func (spotify *Spotify) loadNewReleases(playlist *sconsify.Playlist, webApiCache *WebApiCache) {
	if spotify.client != nil {
		if _, simplePlaylistPage, err := spotify.client.FeaturedPlaylistsOpt(&webspotify.PlaylistOptions{Options: *createWebSpotifyOptions(50, playlist.Playlists())}); err == nil {
			webApiCache.NewReleases = make([]webspotify.FullPlaylist, len(simplePlaylistPage.Playlists))
			for i, webPlaylist := range simplePlaylistPage.Playlists {
				if fullPlaylist, err := spotify.client.GetPlaylist(webPlaylist.Owner.ID, webPlaylist.ID); err == nil {
					webApiCache.NewReleases[i] = *fullPlaylist
				}
			}
		}
	}

	if webApiCache.NewReleases != nil {
		for _, fullPlaylist := range webApiCache.NewReleases {
			tracks := make([]*sconsify.Track, len(fullPlaylist.Tracks.Tracks))
			for i, track := range fullPlaylist.Tracks.Tracks {
				webArtist := track.Track.Artists[0]
				artist := sconsify.InitArtist(string(webArtist.URI), webArtist.Name)
				tracks[i] = sconsify.InitWebApiTrack(string(track.Track.URI), artist, track.Track.Name, track.Track.TimeDuration().String())
			}
			playlist.AddPlaylist(sconsify.InitSubPlaylist(string(fullPlaylist.URI), fullPlaylist.Name, tracks))
		}
		playlist.OpenFolder()
	}
}

func createWebSpotifyOptions(limit int, offset int) *webspotify.Options {
	return &webspotify.Options{Limit: &limit, Offset: &offset}
}

func (spotify *Spotify) initTrack(playlistTrack *sp.PlaylistTrack) *sconsify.Track {
	track := playlistTrack.Track()
	track.Wait()
	for i := 0; i < track.Artists(); i++ {
		track.Artist(i).Wait()
	}
	infrastructure.Debugf("\tTrack '%v' (%v)", track.Link().String(), track.Name())
	return sconsify.ToSconsifyTrack(track)
}

func (spotify *Spotify) canAddPlaylist(playlist *sp.Playlist, playlistType sp.PlaylistType) bool {
	return playlistType == sp.PlaylistTypePlaylist && spotify.isOnFilter(playlist.Name())
}

func (spotify *Spotify) isOnFilter(playlist string) bool {
	if spotify.playlistFilter == nil {
		return true
	}
	for _, filter := range spotify.playlistFilter {
		if filter == playlist {
			return true
		}
	}
	return false
}

func (spotify *Spotify) setPlaylistFilter(playlistFilter string) {
	if playlistFilter == "" {
		return
	}
	spotify.playlistFilter = strings.Split(playlistFilter, ",")
	for i := range spotify.playlistFilter {
		spotify.playlistFilter[i] = strings.Trim(spotify.playlistFilter[i], " ")
	}
}
<|endoftext|>


<commit_before>package config

import (
	"errors"
	"fmt"
	"os"
	"syscall"
)

type closedError struct {
	flockErr error
	fileErr  error
}

func (ce *closedError) Error() string {
	return fmt.Sprintf("%s, %s", ce.fileErr.Error(), ce.flockErr.Error())
}

func newClosedError(flockErr, fileErr error) error {
	if fileErr == nil {
		fileErr = errors.New("no file errors")
	}

	if flockErr == nil {
		flockErr = errors.New("no lock errors")
	}

	return &closedError{flockErr, fileErr}
}

func createOrOpenLockedFile(name string) (file *os.File, err error) {
	if _, serr := os.Stat(name); os.IsNotExist(serr) {
		file, err = os.Create(name)
	} else {
		file, err = os.OpenFile(name, os.O_RDWR, 0644)
	}
	if err != nil {
		return
	}

	if flockErr := syscall.Flock(int(file.Fd()), syscall.LOCK_EX); flockErr != nil {
		err = flockErr
	}

	return
}

func closeLockedFile(file *os.File) error {
	flockErr := syscall.Flock(int(file.Fd()), syscall.LOCK_UN)
	fileErr := file.Close()

	if flockErr != nil || fileErr != nil {
		return newClosedError(flockErr, fileErr)
	}

	return nil
}
<commit_msg>Refactor the internal newClosedError usage<commit_after>package config

import (
	"errors"
	"fmt"
	"os"
	"syscall"
)

type closedError struct {
	flockErr error
	fileErr  error
}

func (ce *closedError) Error() string {
	return fmt.Sprintf("%s, %s", ce.fileErr.Error(), ce.flockErr.Error())
}

func newClosedError(flockErr, fileErr error) error {
	if flockErr == nil && fileErr == nil {
		return nil
	}

	if fileErr == nil {
		fileErr = errors.New("no file errors")
	}

	if flockErr == nil {
		flockErr = errors.New("no lock errors")
	}

	return &closedError{flockErr, fileErr}
}

func createOrOpenLockedFile(name string) (file *os.File, err error) {
	if _, serr := os.Stat(name); os.IsNotExist(serr) {
		file, err = os.Create(name)
	} else {
		file, err = os.OpenFile(name, os.O_RDWR, 0644)
	}
	if err != nil {
		return
	}

	if ferr := syscall.Flock(int(file.Fd()), syscall.LOCK_EX); ferr != nil {
		err = ferr
	}

	return
}

func closeLockedFile(file *os.File) error {
	flockErr := syscall.Flock(int(file.Fd()), syscall.LOCK_UN)
	fileErr := file.Close()

	return newClosedError(flockErr, fileErr)
}
<|endoftext|>


<commit_before>package git

// everything git and github related

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"
)

const GIT_BASE_DIR = "repo"

// Invoke a `command` in `workdir` with `args`, connecting up its Stdout and Stderr
func Command(workdir, command string, args ...string) *exec.Cmd {
	// log.Printf("wd = %s cmd = %s, args = %q", workdir, command, append([]string{}, args...))

	cmd := exec.Command(command, args...)
	cmd.Dir = workdir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd
}

var (
	ErrEmptyRepoName         = errors.New("Empty repository name")
	ErrEmptyRepoOrganization = errors.New("Empty repository organization")
	ErrUserNotAllowed        = errors.New("User not in the allowed set")
)

type Repository struct {
	Name         string `json:"name"`
	Url          string `json:"url"`
	Organization string `json:"organization"`
}

type Pusher struct {
	Name string `json:"name"`
}

type NonGithub struct {
	NoBuild bool `json:"nobuild"`
	Wait    bool `json:"wait"`
}

type JustNongithub struct {
	NonGithub NonGithub `json:"nongithub"`
}

func ParseJustNongithub(in []byte) (j JustNongithub, err error) {
	err = json.Unmarshal(in, &j)
	return
}

type PushEvent struct {
	Ref        string     `json:"ref"`
	Deleted    bool       `json:"deleted"`
	Repository Repository `json:"repository"`
	After      string     `json:"after"`
	Pusher     Pusher     `json:"pusher"`
	NonGithub  NonGithub  `json:"nongithub"`
	HtmlUrl    string     `json:"html_url"`
}

type GithubStatus struct {
	State       string `json:"state"`
	TargetUrl   string `json:"target_url"`
	Description string `json:"description"`
}

var ErrSkipGithubEndpoint = errors.New("Github endpoint skipped")

// Creates or updates a mirror of `url` at `gitDir` using `git clone --mirror`
func LocalMirror(
	url, gitDir, ref string,
	messages io.Writer,
) (err error) {

	// When mirroring, allow up to two minutes before giving up.
	const MirrorTimeout = 2 * time.Minute
	ctx, done := context.WithTimeout(context.Background(), MirrorTimeout)
	defer done()

	if _, err := os.Stat(gitDir); err == nil {
		// Repo already exists, don't need to clone it.

		if AlreadyHaveRef(gitDir, ref) {
			// Sha already exists, don't need to fetch.
			// log.Printf("Already have ref: %v %v", gitDir, ref)
			return nil
		}

		return Fetch(ctx, gitDir, url, messages)
	}

	err = os.MkdirAll(filepath.Dir(gitDir), 0777)
	if err != nil {
		return err
	}

	return Clone(ctx, url, gitDir, messages)
}

func Clone(
	ctx context.Context,
	url, gitDir string,
	messages io.Writer,
) error {
	cmd := Command(".", "git", "clone", "-q", "--mirror", url, gitDir)
	cmd.Stdout = messages
	cmd.Stderr = messages
	return ContextRun(ctx, cmd)
}

// Run cmd within a net Context.
// If the context is cancelled or times out, the process is killed.
func ContextRun(ctx context.Context, cmd *exec.Cmd) error {
	errc := make(chan error)

	err := cmd.Start()
	if err != nil {
		return err
	}

	go func() { errc <- cmd.Wait() }()

	select {
	case <-ctx.Done():
		cmd.Process.Kill()
		return ctx.Err()
	case err := <-errc:
		return err // err may be nil
	}
	return nil
}

func Fetch(
	ctx context.Context,
	gitDir, url string,
	messages io.Writer,
) (err error) {

	cmd := Command(gitDir, "git", "fetch", "-f", url, "*:*")
	cmd.Stdout = messages
	cmd.Stderr = messages

	err = ContextRun(ctx, cmd)
	if err != nil {
		// git fetch where there is no update is exit status 1.
		if err.Error() != "exit status 1" {
			return err
		}
	}

	return nil
}

var ShaLike = regexp.MustCompile("[0-9a-zA-Z]{40}")

// Returns true if ref is sha-like and is in the object database.
// The "sha-like" condition ensures that refs like `master` are always
// freshened.
func AlreadyHaveRef(gitDir, sha string) bool {
	if !ShaLike.MatchString(sha) {
		return false
	}
	cmd := Command(gitDir, "git", "cat-file", "-t", sha)
	cmd.Stdout = nil
	cmd.Stderr = nil

	err := cmd.Run()
	return err == nil
}

func HaveFile(gitDir, ref, path string) (ok bool, err error) {
	cmd := Command(gitDir, "git", "show", fmt.Sprintf("%s:%s", ref, path))
	cmd.Stdout = nil // don't want to see the contents
	err = cmd.Run()
	ok = true
	if err != nil {
		ok = false
		if err.Error() == "exit status 128" {
			// This happens if the file doesn't exist.
			err = nil
		}
	}
	return ok, err
}

func RevParse(gitDir, ref string) (sha string, err error) {
	cmd := Command(gitDir, "git", "rev-parse", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return
	}

	sha = strings.TrimSpace(string(stdout))
	return
}

func Describe(gitDir, ref string) (desc string, err error) {
	cmd := Command(gitDir, "git", "describe", "--all", "--tags", "--long", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return
	}

	desc = strings.TrimSpace(string(stdout))
	desc = strings.TrimPrefix(desc, "heads/")
	return
}

func Checkout(gitDir, checkoutDir, ref string) error {

	err := os.MkdirAll(checkoutDir, 0777)
	if err != nil {
		return err
	}

	args := []string{"--work-tree", checkoutDir, "checkout", ref, "--", "."}
	err = Command(gitDir, "git", args...).Run()
	if err != nil {
		return err
	}

	// Set mtimes to time file is most recently affected by a commit.
	// This is annoying but unfortunately git sets the timestamps to now,
	// and docker depends on the mtime for cache invalidation.
	err = SetMTimes(gitDir, checkoutDir, ref)
	if err != nil {
		return err
	}

	return nil
}

type BuildDirectory struct {
	Name, Dir string
	Cleanup   func()
}

func PrepBuildDirectory(
	gitDir, remote, ref string,
) (*BuildDirectory, error) {

	start := time.Now()
	defer func() {
		log.Printf("Took %v to prep %v", time.Since(start), remote)
	}()

	if strings.HasPrefix(remote, "github.com/") {
		remote = "https://" + remote
	}

	gitDir, err := filepath.Abs(gitDir)
	if err != nil {
		return nil, fmt.Errorf("unable to determine abspath: %v", err)
	}

	err = LocalMirror(remote, gitDir, ref, os.Stderr)
	if err != nil {
		return nil, fmt.Errorf("unable to LocalMirror: %v", err)
	}

	rev, err := RevParse(gitDir, ref)
	if err != nil {
		return nil, fmt.Errorf("unable to parse rev: %v", err)
	}

	tagName, err := Describe(gitDir, rev)
	if err != nil {
		return nil, fmt.Errorf("unable to describe %v: %v", rev, err)
	}

	shortRev := rev[:10]
	checkoutPath := path.Join(gitDir, filepath.Join("c/", shortRev))

	err = recursiveCheckout(gitDir, checkoutPath, rev)
	if err != nil {
		return nil, err
	}

	cleanup := func() {
		err := SafeCleanup(checkoutPath)
		if err != nil {
			log.Printf("Error cleaning up path: ", checkoutPath)
		}
	}

	return &BuildDirectory{tagName, checkoutPath, cleanup}, nil
}

func recursiveCheckout(gitDir, checkoutPath, rev string) error {
	err := Checkout(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to checkout: %v", err)
	}

	err = PrepSubmodules(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to prep submodules: %v", err)
	}
	return nil
}

func SafeCleanup(path string) error {
	if path == "/" || path == "" || path == "." || strings.Contains(path, "..") {
		return fmt.Errorf("invalid path specified for deletion %q", path)
	}
	return os.RemoveAll(path)
}
<commit_msg>Don't use stdout for any executed commands<commit_after>package git

// everything git and github related

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"
)

const GIT_BASE_DIR = "repo"

// Invoke a `command` in `workdir` with `args`, connecting Stdout and Stderr to Stderr.
func Command(workdir, command string, args ...string) *exec.Cmd {
	// log.Printf("wd = %s cmd = %s, args = %q", workdir, command, append([]string{}, args...))

	cmd := exec.Command(command, args...)
	cmd.Dir = workdir
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	return cmd
}

var (
	ErrEmptyRepoName         = errors.New("Empty repository name")
	ErrEmptyRepoOrganization = errors.New("Empty repository organization")
	ErrUserNotAllowed        = errors.New("User not in the allowed set")
)

type Repository struct {
	Name         string `json:"name"`
	Url          string `json:"url"`
	Organization string `json:"organization"`
}

type Pusher struct {
	Name string `json:"name"`
}

type NonGithub struct {
	NoBuild bool `json:"nobuild"`
	Wait    bool `json:"wait"`
}

type JustNongithub struct {
	NonGithub NonGithub `json:"nongithub"`
}

func ParseJustNongithub(in []byte) (j JustNongithub, err error) {
	err = json.Unmarshal(in, &j)
	return
}

type PushEvent struct {
	Ref        string     `json:"ref"`
	Deleted    bool       `json:"deleted"`
	Repository Repository `json:"repository"`
	After      string     `json:"after"`
	Pusher     Pusher     `json:"pusher"`
	NonGithub  NonGithub  `json:"nongithub"`
	HtmlUrl    string     `json:"html_url"`
}

type GithubStatus struct {
	State       string `json:"state"`
	TargetUrl   string `json:"target_url"`
	Description string `json:"description"`
}

var ErrSkipGithubEndpoint = errors.New("Github endpoint skipped")

// Creates or updates a mirror of `url` at `gitDir` using `git clone --mirror`
func LocalMirror(
	url, gitDir, ref string,
	messages io.Writer,
) (err error) {

	// When mirroring, allow up to two minutes before giving up.
	const MirrorTimeout = 2 * time.Minute
	ctx, done := context.WithTimeout(context.Background(), MirrorTimeout)
	defer done()

	if _, err := os.Stat(gitDir); err == nil {
		// Repo already exists, don't need to clone it.

		if AlreadyHaveRef(gitDir, ref) {
			// Sha already exists, don't need to fetch.
			// log.Printf("Already have ref: %v %v", gitDir, ref)
			return nil
		}

		return Fetch(ctx, gitDir, url, messages)
	}

	err = os.MkdirAll(filepath.Dir(gitDir), 0777)
	if err != nil {
		return err
	}

	return Clone(ctx, url, gitDir, messages)
}

func Clone(
	ctx context.Context,
	url, gitDir string,
	messages io.Writer,
) error {
	cmd := Command(".", "git", "clone", "-q", "--mirror", url, gitDir)
	cmd.Stdout = messages
	cmd.Stderr = messages
	return ContextRun(ctx, cmd)
}

// Run cmd within a net Context.
// If the context is cancelled or times out, the process is killed.
func ContextRun(ctx context.Context, cmd *exec.Cmd) error {
	errc := make(chan error)

	err := cmd.Start()
	if err != nil {
		return err
	}

	go func() { errc <- cmd.Wait() }()

	select {
	case <-ctx.Done():
		cmd.Process.Kill()
		return ctx.Err()
	case err := <-errc:
		return err // err may be nil
	}
	return nil
}

func Fetch(
	ctx context.Context,
	gitDir, url string,
	messages io.Writer,
) (err error) {

	cmd := Command(gitDir, "git", "fetch", "-f", url, "*:*")
	cmd.Stdout = messages
	cmd.Stderr = messages

	err = ContextRun(ctx, cmd)
	if err != nil {
		// git fetch where there is no update is exit status 1.
		if err.Error() != "exit status 1" {
			return err
		}
	}

	return nil
}

var ShaLike = regexp.MustCompile("[0-9a-zA-Z]{40}")

// Returns true if ref is sha-like and is in the object database.
// The "sha-like" condition ensures that refs like `master` are always
// freshened.
func AlreadyHaveRef(gitDir, sha string) bool {
	if !ShaLike.MatchString(sha) {
		return false
	}
	cmd := Command(gitDir, "git", "cat-file", "-t", sha)
	cmd.Stdout = nil
	cmd.Stderr = nil

	err := cmd.Run()
	return err == nil
}

func HaveFile(gitDir, ref, path string) (ok bool, err error) {
	cmd := Command(gitDir, "git", "show", fmt.Sprintf("%s:%s", ref, path))
	cmd.Stdout = nil // don't want to see the contents
	err = cmd.Run()
	ok = true
	if err != nil {
		ok = false
		if err.Error() == "exit status 128" {
			// This happens if the file doesn't exist.
			err = nil
		}
	}
	return ok, err
}

func RevParse(gitDir, ref string) (sha string, err error) {
	cmd := Command(gitDir, "git", "rev-parse", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return
	}

	sha = strings.TrimSpace(string(stdout))
	return
}

func Describe(gitDir, ref string) (desc string, err error) {
	cmd := Command(gitDir, "git", "describe", "--all", "--tags", "--long", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return
	}

	desc = strings.TrimSpace(string(stdout))
	desc = strings.TrimPrefix(desc, "heads/")
	return
}

func Checkout(gitDir, checkoutDir, ref string) error {

	err := os.MkdirAll(checkoutDir, 0777)
	if err != nil {
		return err
	}

	args := []string{"--work-tree", checkoutDir, "checkout", ref, "--", "."}
	err = Command(gitDir, "git", args...).Run()
	if err != nil {
		return err
	}

	// Set mtimes to time file is most recently affected by a commit.
	// This is annoying but unfortunately git sets the timestamps to now,
	// and docker depends on the mtime for cache invalidation.
	err = SetMTimes(gitDir, checkoutDir, ref)
	if err != nil {
		return err
	}

	return nil
}

type BuildDirectory struct {
	Name, Dir string
	Cleanup   func()
}

func PrepBuildDirectory(
	gitDir, remote, ref string,
) (*BuildDirectory, error) {

	start := time.Now()
	defer func() {
		log.Printf("Took %v to prep %v", time.Since(start), remote)
	}()

	if strings.HasPrefix(remote, "github.com/") {
		remote = "https://" + remote
	}

	gitDir, err := filepath.Abs(gitDir)
	if err != nil {
		return nil, fmt.Errorf("unable to determine abspath: %v", err)
	}

	err = LocalMirror(remote, gitDir, ref, os.Stderr)
	if err != nil {
		return nil, fmt.Errorf("unable to LocalMirror: %v", err)
	}

	rev, err := RevParse(gitDir, ref)
	if err != nil {
		return nil, fmt.Errorf("unable to parse rev: %v", err)
	}

	tagName, err := Describe(gitDir, rev)
	if err != nil {
		return nil, fmt.Errorf("unable to describe %v: %v", rev, err)
	}

	shortRev := rev[:10]
	checkoutPath := path.Join(gitDir, filepath.Join("c/", shortRev))

	err = recursiveCheckout(gitDir, checkoutPath, rev)
	if err != nil {
		return nil, err
	}

	cleanup := func() {
		err := SafeCleanup(checkoutPath)
		if err != nil {
			log.Printf("Error cleaning up path: ", checkoutPath)
		}
	}

	return &BuildDirectory{tagName, checkoutPath, cleanup}, nil
}

func recursiveCheckout(gitDir, checkoutPath, rev string) error {
	err := Checkout(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to checkout: %v", err)
	}

	err = PrepSubmodules(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to prep submodules: %v", err)
	}
	return nil
}

func SafeCleanup(path string) error {
	if path == "/" || path == "" || path == "." || strings.Contains(path, "..") {
		return fmt.Errorf("invalid path specified for deletion %q", path)
	}
	return os.RemoveAll(path)
}
<|endoftext|>


<commit_before>package operation

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
	"runtime/debug"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/valyala/bytebufferpool"
)

type UploadResult struct {
	Name       string `json:"name,omitempty"`
	Size       uint32 `json:"size,omitempty"`
	Error      string `json:"error,omitempty"`
	ETag       string `json:"eTag,omitempty"`
	CipherKey  []byte `json:"cipherKey,omitempty"`
	Mime       string `json:"mime,omitempty"`
	Gzip       uint32 `json:"gzip,omitempty"`
	ContentMd5 string `json:"contentMd5,omitempty"`
}

func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *filer_pb.FileChunk {
	fid, _ := filer_pb.ToFileIdObject(fileId)
	return &filer_pb.FileChunk{
		FileId:       fileId,
		Offset:       offset,
		Size:         uint64(uploadResult.Size),
		Mtime:        time.Now().UnixNano(),
		ETag:         uploadResult.ETag,
		CipherKey:    uploadResult.CipherKey,
		IsCompressed: uploadResult.Gzip > 0,
		Fid:          fid,
	}
}

// HTTPClient interface for testing
type HTTPClient interface {
	Do(req *http.Request) (*http.Response, error)
}

var (
	HttpClient HTTPClient
)

func init() {
	HttpClient = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	uploadResult, err = retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	return
}

// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	uploadResult, err, data = doUpload(uploadUrl, filename, cipher, reader, isInputCompressed, mtype, pairMap, jwt)
	return
}

func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	bytesReader, ok := reader.(*util.BytesReader)
	if ok {
		data = bytesReader.Bytes
	} else {
		buf := bytebufferpool.Get()
		_, err = buf.ReadFrom(reader)
		defer bytebufferpool.Put(buf)
		if err != nil {
			err = fmt.Errorf("read input: %v", err)
			return
		}
		data = buf.Bytes()
	}
	uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	return uploadResult, uploadErr, data
}

func retriedUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	for i := 0; i < 1; i++ {
		uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
		if err == nil {
			return
		} else {
			glog.Warningf("uploading to %s: %v", uploadUrl, err)
		}
	}
	return
}

func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	contentIsGzipped := isInputCompressed
	shouldGzipNow := false
	if !isInputCompressed {
		if mtype == "" {
			mtype = http.DetectContentType(data)
			// println("detect1 mimetype to", mtype)
			if mtype == "application/octet-stream" {
				mtype = ""
			}
		}
		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed {
			shouldGzipNow = true
		} else if !iAmSure && mtype == "" && len(data) > 16*1024 {
			var compressed []byte
			compressed, err = util.GzipData(data[0:128])
			shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
		}
	}

	var clearDataLen int

	// gzip if possible
	// this could be double copying
	clearDataLen = len(data)
	if shouldGzipNow {
		compressed, compressErr := util.GzipData(data)
		// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
		if compressErr == nil {
			data = compressed
			contentIsGzipped = true
		}
	} else if isInputCompressed {
		// just to get the clear data length
		clearData, err := util.DecompressData(data)
		if err == nil {
			clearDataLen = len(clearData)
		}
	}

	if cipher {
		// encrypt(gzip(data))

		// encrypt
		cipherKey := util.GenCipherKey()
		encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
		if encryptionErr != nil {
			err = fmt.Errorf("encrypt input: %v", encryptionErr)
			return
		}

		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(encryptedData)
			return
		}, "", false, len(encryptedData), "", nil, jwt)
		if uploadResult != nil {
			uploadResult.Name = filename
			uploadResult.Mime = mtype
			uploadResult.CipherKey = cipherKey
		}
	} else {
		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(data)
			return
		}, filename, contentIsGzipped, 0, mtype, pairMap, jwt)
	}

	if uploadResult == nil {
		return
	}

	uploadResult.Size = uint32(clearDataLen)
	if contentIsGzipped {
		uploadResult.Gzip = 1
	}

	return uploadResult, err
}

func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	buf := bytebufferpool.Get()
	defer bytebufferpool.Put(buf)
	body_writer := multipart.NewWriter(buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
	if mtype == "" {
		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
	}
	if mtype != "" {
		h.Set("Content-Type", mtype)
	}
	if isGzipped {
		h.Set("Content-Encoding", "gzip")
	}

	file_writer, cp_err := body_writer.CreatePart(h)
	if cp_err != nil {
		glog.V(0).Infoln("error creating form file", cp_err.Error())
		return nil, cp_err
	}
	if err := fillBufferFunction(file_writer); err != nil {
		glog.V(0).Infoln("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err := body_writer.Close(); err != nil {
		glog.V(0).Infoln("error closing body", err)
		return nil, err
	}

	req, postErr := http.NewRequest("POST", uploadUrl, bytes.NewReader(buf.Bytes()))
	if postErr != nil {
		glog.V(1).Infof("create upload request %s: %v", uploadUrl, postErr)
		return nil, fmt.Errorf("create upload request %s: %v", uploadUrl, postErr)
	}
	req.Header.Set("Content-Type", content_type)
	for k, v := range pairMap {
		req.Header.Set(k, v)
	}
	if jwt != "" {
		req.Header.Set("Authorization", "BEARER "+string(jwt))
	}
	// print("+")
	resp, post_err := HttpClient.Do(req)
	if post_err != nil {
		glog.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
		debug.PrintStack()
		return nil, fmt.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
	}
	// print("-")
	defer util.CloseResponse(resp)

	var ret UploadResult
	etag := getEtag(resp)
	if resp.StatusCode == http.StatusNoContent {
		ret.ETag = etag
		return &ret, nil
	}

	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		return nil, fmt.Errorf("read response body %v: %v", uploadUrl, ra_err)
	}

	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.Errorf("unmarshal %s: %v", uploadUrl, string(resp_body))
		return nil, fmt.Errorf("unmarshal %v: %v", uploadUrl, unmarshal_err)
	}
	if ret.Error != "" {
		return nil, fmt.Errorf("unmarshalled error %v: %v", uploadUrl, ret.Error)
	}
	ret.ETag = etag
	ret.ContentMd5 = resp.Header.Get("Content-MD5")
	return &ret, nil
}

func getEtag(r *http.Response) (etag string) {
	etag = r.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		etag = etag[1 : len(etag)-1]
	}
	return
}
<commit_msg>add back upload retries<commit_after>package operation

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
	"runtime/debug"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/valyala/bytebufferpool"
)

type UploadResult struct {
	Name       string `json:"name,omitempty"`
	Size       uint32 `json:"size,omitempty"`
	Error      string `json:"error,omitempty"`
	ETag       string `json:"eTag,omitempty"`
	CipherKey  []byte `json:"cipherKey,omitempty"`
	Mime       string `json:"mime,omitempty"`
	Gzip       uint32 `json:"gzip,omitempty"`
	ContentMd5 string `json:"contentMd5,omitempty"`
}

func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *filer_pb.FileChunk {
	fid, _ := filer_pb.ToFileIdObject(fileId)
	return &filer_pb.FileChunk{
		FileId:       fileId,
		Offset:       offset,
		Size:         uint64(uploadResult.Size),
		Mtime:        time.Now().UnixNano(),
		ETag:         uploadResult.ETag,
		CipherKey:    uploadResult.CipherKey,
		IsCompressed: uploadResult.Gzip > 0,
		Fid:          fid,
	}
}

// HTTPClient interface for testing
type HTTPClient interface {
	Do(req *http.Request) (*http.Response, error)
}

var (
	HttpClient HTTPClient
)

func init() {
	HttpClient = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	uploadResult, err = retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	return
}

// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	uploadResult, err, data = doUpload(uploadUrl, filename, cipher, reader, isInputCompressed, mtype, pairMap, jwt)
	return
}

func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	bytesReader, ok := reader.(*util.BytesReader)
	if ok {
		data = bytesReader.Bytes
	} else {
		buf := bytebufferpool.Get()
		_, err = buf.ReadFrom(reader)
		defer bytebufferpool.Put(buf)
		if err != nil {
			err = fmt.Errorf("read input: %v", err)
			return
		}
		data = buf.Bytes()
	}
	uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	return uploadResult, uploadErr, data
}

func retriedUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	for i := 0; i < 3; i++ {
		uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
		if err == nil {
			return
		} else {
			glog.Warningf("uploading to %s: %v", uploadUrl, err)
		}
	}
	return
}

func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	contentIsGzipped := isInputCompressed
	shouldGzipNow := false
	if !isInputCompressed {
		if mtype == "" {
			mtype = http.DetectContentType(data)
			// println("detect1 mimetype to", mtype)
			if mtype == "application/octet-stream" {
				mtype = ""
			}
		}
		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed {
			shouldGzipNow = true
		} else if !iAmSure && mtype == "" && len(data) > 16*1024 {
			var compressed []byte
			compressed, err = util.GzipData(data[0:128])
			shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
		}
	}

	var clearDataLen int

	// gzip if possible
	// this could be double copying
	clearDataLen = len(data)
	if shouldGzipNow {
		compressed, compressErr := util.GzipData(data)
		// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
		if compressErr == nil {
			data = compressed
			contentIsGzipped = true
		}
	} else if isInputCompressed {
		// just to get the clear data length
		clearData, err := util.DecompressData(data)
		if err == nil {
			clearDataLen = len(clearData)
		}
	}

	if cipher {
		// encrypt(gzip(data))

		// encrypt
		cipherKey := util.GenCipherKey()
		encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
		if encryptionErr != nil {
			err = fmt.Errorf("encrypt input: %v", encryptionErr)
			return
		}

		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(encryptedData)
			return
		}, "", false, len(encryptedData), "", nil, jwt)
		if uploadResult != nil {
			uploadResult.Name = filename
			uploadResult.Mime = mtype
			uploadResult.CipherKey = cipherKey
		}
	} else {
		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(data)
			return
		}, filename, contentIsGzipped, 0, mtype, pairMap, jwt)
	}

	if uploadResult == nil {
		return
	}

	uploadResult.Size = uint32(clearDataLen)
	if contentIsGzipped {
		uploadResult.Gzip = 1
	}

	return uploadResult, err
}

func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	buf := bytebufferpool.Get()
	defer bytebufferpool.Put(buf)
	body_writer := multipart.NewWriter(buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
	if mtype == "" {
		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
	}
	if mtype != "" {
		h.Set("Content-Type", mtype)
	}
	if isGzipped {
		h.Set("Content-Encoding", "gzip")
	}

	file_writer, cp_err := body_writer.CreatePart(h)
	if cp_err != nil {
		glog.V(0).Infoln("error creating form file", cp_err.Error())
		return nil, cp_err
	}
	if err := fillBufferFunction(file_writer); err != nil {
		glog.V(0).Infoln("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err := body_writer.Close(); err != nil {
		glog.V(0).Infoln("error closing body", err)
		return nil, err
	}

	req, postErr := http.NewRequest("POST", uploadUrl, bytes.NewReader(buf.Bytes()))
	if postErr != nil {
		glog.V(1).Infof("create upload request %s: %v", uploadUrl, postErr)
		return nil, fmt.Errorf("create upload request %s: %v", uploadUrl, postErr)
	}
	req.Header.Set("Content-Type", content_type)
	for k, v := range pairMap {
		req.Header.Set(k, v)
	}
	if jwt != "" {
		req.Header.Set("Authorization", "BEARER "+string(jwt))
	}
	// print("+")
	resp, post_err := HttpClient.Do(req)
	if post_err != nil {
		glog.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
		debug.PrintStack()
		return nil, fmt.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
	}
	// print("-")
	defer util.CloseResponse(resp)

	var ret UploadResult
	etag := getEtag(resp)
	if resp.StatusCode == http.StatusNoContent {
		ret.ETag = etag
		return &ret, nil
	}

	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		return nil, fmt.Errorf("read response body %v: %v", uploadUrl, ra_err)
	}

	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.Errorf("unmarshal %s: %v", uploadUrl, string(resp_body))
		return nil, fmt.Errorf("unmarshal %v: %v", uploadUrl, unmarshal_err)
	}
	if ret.Error != "" {
		return nil, fmt.Errorf("unmarshalled error %v: %v", uploadUrl, ret.Error)
	}
	ret.ETag = etag
	ret.ContentMd5 = resp.Header.Get("Content-MD5")
	return &ret, nil
}

func getEtag(r *http.Response) (etag string) {
	etag = r.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		etag = etag[1 : len(etag)-1]
	}
	return
}
<|endoftext|>


<commit_before>package goconfig

import (
	"testing"
)

func TestConfigImp(t *testing.T) {
	config_mng := NewConfig()
	config_mng.Reg("info", "test_config_obj.json", true)

	data := config_mng.Get("info")

	t.Log("config : ", data)
	t.Log("config : ", data.Get("info1"))
	t.Log("config : ", data.Get("info2"))
	t.Log("config : ", data.Get("info3"))
	t.Log("config : ", data.Get("info_arr"))
	t.Log("config : ", data.Get("info_obj"))
	t.Log("config : ", data.Get("info_arr").GetIndex(2))
	t.Log("config : ", data.Get("info_obj").Get("o1"))

	t.Log("config : ", data.Get("info_obj").Get("o2").GetIndex(4))
	t.Log("config : ", data.Get("info_obj333").Get("o2").GetIndex(4))

	if data.Get("info3").MustString() != "aaa3" {
		t.Error("info error!")
	}

	config_mng.Close()

	return
}

func TestConfigGetByIdx(t *testing.T) {
	config_mng := NewConfig()
	config_mng.Reg("info", "test_config_array.json", true)

	data1 := config_mng.GetByIdx("info", "1")
	data2 := config_mng.GetByIdx("info", "2")
	datanil := config_mng.GetByIdx("info", "9999")

	t.Log("config : ", data1)
	t.Log("config : ", data2)
	t.Log("config : ", datanil)

	if data1.Get("Info").MustString() != "Info1" {
		t.Error("data1 error!")
	}
	if data2.Get("Info").MustString() != "Info2" {
		t.Error("data1 error!")
	}

	config_mng.Close()

	return
}

func TestConfigUpdate(t *testing.T) {
	update_chan := make(chan string)

	config_mng := NewConfig()
	config_mng.Reg("info", "test_config_obj.json", true)
	config_mng.RegNotifyChan(update_chan)

	data := config_mng.Get("info")

	t.Log("config : ", data)

	for {
		select {
		case update_name, ok := <-update_chan:
			if !ok {
				return
			}
			t.Log("config : ", update_name)
			data = config_mng.Get("info")
			goto RETURN_INFO
		}
	}

RETURN_INFO:
	config_mng.Close()

	t.Log("config : ", data)
	t.Log("config : ", data.Get("info1"))
	t.Log("config : ", data.Get("info2"))
	t.Log("config : ", data.Get("info3"))
	t.Log("config : ", data.Get("info_arr"))
	t.Log("config : ", data.Get("info_obj"))
	t.Log("config : ", data.Get("info_arr").GetIndex(2))
	t.Log("config : ", data.Get("info_obj").Get("o1"))

	t.Log("config : ", data.Get("info_obj").Get("o2").GetIndex(4))
	t.Log("config : ", data.Get("info_obj333").Get("o2").GetIndex(4))

	return
}
<commit_msg>change test<commit_after>package goconfig

import (
	"bufio"
	"os"
	"testing"
	"time"
)

func changeConfigFile(path string) {
	file, err := os.Create(path)
	if err != nil {
		return
	}
	defer file.Close()

	writer := bufio.NewWriter(file)

	writer.WriteString(`{
		"info1" : 333,
		"info2" : "aaa2",
		"info3" : "aaa3",
		"info_arr" : ["a1", "a2", "a3"],
		"info_obj" : {
			"o1" : "sss",
			"o2" : ["o21", "o22"]
			}
	}`)
	writer.Flush()

}

func TestConfigImp(t *testing.T) {
	config_mng := NewConfig()
	config_mng.Reg("info", "test_config_obj.json", true)

	data := config_mng.Get("info")

	t.Log("config : ", data)
	t.Log("config : ", data.Get("info1"))
	t.Log("config : ", data.Get("info2"))
	t.Log("config : ", data.Get("info3"))
	t.Log("config : ", data.Get("info_arr"))
	t.Log("config : ", data.Get("info_obj"))
	t.Log("config : ", data.Get("info_arr").GetIndex(2))
	t.Log("config : ", data.Get("info_obj").Get("o1"))

	t.Log("config : ", data.Get("info_obj").Get("o2").GetIndex(4))
	t.Log("config : ", data.Get("info_obj333").Get("o2").GetIndex(4))

	if data.Get("info3").MustString() != "aaa3" {
		t.Error("info error!")
	}

	config_mng.Close()

	return
}

func TestConfigGetByIdx(t *testing.T) {
	config_mng := NewConfig()
	config_mng.Reg("info", "test_config_array.json", true)

	data1 := config_mng.GetByIdx("info", "1")
	data2 := config_mng.GetByIdx("info", "2")
	datanil := config_mng.GetByIdx("info", "9999")

	t.Log("config : ", data1)
	t.Log("config : ", data2)
	t.Log("config : ", datanil)

	if data1.Get("Info").MustString() != "Info1" {
		t.Error("data1 error!")
	}
	if data2.Get("Info").MustString() != "Info2" {
		t.Error("data1 error!")
	}

	config_mng.Close()

	return
}

func TestConfigUpdate(t *testing.T) {
	update_chan := make(chan string)

	config_mng := NewConfig()
	config_mng.Reg("info", "test_config_obj.json", true)
	config_mng.RegNotifyChan(update_chan)

	data := config_mng.Get("info")

	t.Log("config : ", data)

	go func() {
		time.Sleep(2 * time.Second)
		changeConfigFile("test_config_obj.json")
	}()

	for {
		select {
		case update_name, ok := <-update_chan:
			if !ok {
				return
			}
			t.Log("config : ", update_name)
			data = config_mng.Get("info")
			goto RETURN_INFO
		}
	}

RETURN_INFO:
	config_mng.Close()

	t.Log("config : ", data)
	t.Log("config : ", data.Get("info1"))
	t.Log("config : ", data.Get("info2"))
	t.Log("config : ", data.Get("info3"))
	t.Log("config : ", data.Get("info_arr"))
	t.Log("config : ", data.Get("info_obj"))
	t.Log("config : ", data.Get("info_arr").GetIndex(2))
	t.Log("config : ", data.Get("info_obj").Get("o1"))

	t.Log("config : ", data.Get("info_obj").Get("o2").GetIndex(4))
	t.Log("config : ", data.Get("info_obj333").Get("o2").GetIndex(4))

	return
}
<|endoftext|>


<commit_before>package authn

import (
	"strings"
	"sync"

	"github.com/pkg/errors"
	"github.com/rancher/norman/httperror"
	"github.com/rancher/norman/store/transform"
	"github.com/rancher/norman/types"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/client/management/v3"
	"github.com/rancher/types/config"
	"golang.org/x/crypto/bcrypt"
	"k8s.io/client-go/tools/cache"
)

const userByUsernameIndex = "auth.management.cattle.io/user-by-username"

type userStore struct {
	types.Store
	mu          sync.Mutex
	userIndexer cache.Indexer
}

func SetUserStore(schema *types.Schema, mgmt *config.ScaledContext) {
	userInformer := mgmt.Management.Users("").Controller().Informer()
	userIndexers := map[string]cache.IndexFunc{
		userByUsernameIndex: userByUsername,
	}
	userInformer.AddIndexers(userIndexers)

	store := &userStore{
		Store:       schema.Store,
		mu:          sync.Mutex{},
		userIndexer: userInformer.GetIndexer(),
	}

	t := &transform.Store{
		Store: store,
		Transformer: func(apiContext *types.APIContext, data map[string]interface{}, opt *types.QueryOptions) (map[string]interface{}, error) {
			// filter system users out of the api
			if princIds, ok := data[client.UserFieldPrincipalIDs].([]interface{}); ok {
				for _, p := range princIds {
					pid, _ := p.(string)
					if strings.HasPrefix(pid, "system://") {
						if opt != nil && opt.Options["ByID"] == "true" {
							return nil, httperror.NewAPIError(httperror.NotFound, "resource not found")
						}
					}
				}
			}

			// set "me" field on user
			userID := apiContext.Request.Header.Get("Impersonate-User")
			if userID != "" {
				id, ok := data[types.ResourceFieldID].(string)
				if ok {
					if id == userID {
						data["me"] = "true"
					}
				}
			}

			return data, nil
		},
	}

	schema.Store = t
}

func userByUsername(obj interface{}) ([]string, error) {
	u, ok := obj.(*v3.User)
	if !ok {
		return []string{}, nil
	}

	return []string{u.Username}, nil
}

func hashPassword(data map[string]interface{}) error {
	pass, ok := data[client.UserFieldPassword].(string)
	if !ok {
		return errors.New("password not a string")
	}
	hashed, err := hashPasswordString(pass)
	if err != nil {
		return err
	}
	data[client.UserFieldPassword] = string(hashed)

	return nil
}

func hashPasswordString(password string) (string, error) {
	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
	if err != nil {
		return "", errors.Wrap(err, "problem encrypting password")
	}
	return string(hash), nil
}

func (s *userStore) Create(apiContext *types.APIContext, schema *types.Schema, data map[string]interface{}) (map[string]interface{}, error) {
	if err := hashPassword(data); err != nil {
		return nil, err
	}

	created, err := s.create(apiContext, schema, data)
	if err != nil {
		return nil, err
	}

	if id, ok := created[types.ResourceFieldID].(string); ok {
		var principalIDs []interface{}
		if pids, ok := created[client.UserFieldPrincipalIDs].([]interface{}); ok {
			principalIDs = pids
		}
		created[client.UserFieldPrincipalIDs] = append(principalIDs, "local://"+id)
		return s.Update(apiContext, schema, created, id)
	}

	return created, err
}

func (s *userStore) create(apiContext *types.APIContext, schema *types.Schema, data map[string]interface{}) (map[string]interface{}, error) {
	username, ok := data[client.UserFieldUsername].(string)
	if !ok {
		return nil, errors.New("invalid username")
	}

	s.mu.Lock()
	defer s.mu.Unlock()

	users, err := s.userIndexer.ByIndex(userByUsernameIndex, username)
	if err != nil {
		return nil, err
	}
	if len(users) > 0 {
		return nil, httperror.NewFieldAPIError(httperror.NotUnique, "username", "Username is already in use.")
	}

	return s.Store.Create(apiContext, schema, data)
}
<commit_msg>dont return system user in list<commit_after>package authn

import (
	"strings"
	"sync"

	"github.com/pkg/errors"
	"github.com/rancher/norman/httperror"
	"github.com/rancher/norman/store/transform"
	"github.com/rancher/norman/types"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/client/management/v3"
	"github.com/rancher/types/config"
	"golang.org/x/crypto/bcrypt"
	"k8s.io/client-go/tools/cache"
)

const userByUsernameIndex = "auth.management.cattle.io/user-by-username"

type userStore struct {
	types.Store
	mu          sync.Mutex
	userIndexer cache.Indexer
}

func SetUserStore(schema *types.Schema, mgmt *config.ScaledContext) {
	userInformer := mgmt.Management.Users("").Controller().Informer()
	userIndexers := map[string]cache.IndexFunc{
		userByUsernameIndex: userByUsername,
	}
	userInformer.AddIndexers(userIndexers)

	store := &userStore{
		Store:       schema.Store,
		mu:          sync.Mutex{},
		userIndexer: userInformer.GetIndexer(),
	}

	t := &transform.Store{
		Store: store,
		Transformer: func(apiContext *types.APIContext, data map[string]interface{}, opt *types.QueryOptions) (map[string]interface{}, error) {
			// filter system users out of the api
			if princIds, ok := data[client.UserFieldPrincipalIDs].([]interface{}); ok {
				for _, p := range princIds {
					pid, _ := p.(string)
					if strings.HasPrefix(pid, "system://") {
						if opt != nil && opt.Options["ByID"] == "true" {
							return nil, httperror.NewAPIError(httperror.NotFound, "resource not found")
						}
						return nil, nil
					}
				}
			}

			// set "me" field on user
			userID := apiContext.Request.Header.Get("Impersonate-User")
			if userID != "" {
				id, ok := data[types.ResourceFieldID].(string)
				if ok {
					if id == userID {
						data["me"] = "true"
					}
				}
			}

			return data, nil
		},
	}

	schema.Store = t
}

func userByUsername(obj interface{}) ([]string, error) {
	u, ok := obj.(*v3.User)
	if !ok {
		return []string{}, nil
	}

	return []string{u.Username}, nil
}

func hashPassword(data map[string]interface{}) error {
	pass, ok := data[client.UserFieldPassword].(string)
	if !ok {
		return errors.New("password not a string")
	}
	hashed, err := hashPasswordString(pass)
	if err != nil {
		return err
	}
	data[client.UserFieldPassword] = string(hashed)

	return nil
}

func hashPasswordString(password string) (string, error) {
	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
	if err != nil {
		return "", errors.Wrap(err, "problem encrypting password")
	}
	return string(hash), nil
}

func (s *userStore) Create(apiContext *types.APIContext, schema *types.Schema, data map[string]interface{}) (map[string]interface{}, error) {
	if err := hashPassword(data); err != nil {
		return nil, err
	}

	created, err := s.create(apiContext, schema, data)
	if err != nil {
		return nil, err
	}

	if id, ok := created[types.ResourceFieldID].(string); ok {
		var principalIDs []interface{}
		if pids, ok := created[client.UserFieldPrincipalIDs].([]interface{}); ok {
			principalIDs = pids
		}
		created[client.UserFieldPrincipalIDs] = append(principalIDs, "local://"+id)
		return s.Update(apiContext, schema, created, id)
	}

	return created, err
}

func (s *userStore) create(apiContext *types.APIContext, schema *types.Schema, data map[string]interface{}) (map[string]interface{}, error) {
	username, ok := data[client.UserFieldUsername].(string)
	if !ok {
		return nil, errors.New("invalid username")
	}

	s.mu.Lock()
	defer s.mu.Unlock()

	users, err := s.userIndexer.ByIndex(userByUsernameIndex, username)
	if err != nil {
		return nil, err
	}
	if len(users) > 0 {
		return nil, httperror.NewFieldAPIError(httperror.NotUnique, "username", "Username is already in use.")
	}

	return s.Store.Create(apiContext, schema, data)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vfsclientset

import (
	"fmt"
	"os"
	"strings"
	"time"

	apiequality "k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/klog"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/registry"
	"k8s.io/kops/pkg/apis/kops/v1alpha1"
	"k8s.io/kops/pkg/apis/kops/validation"
	"k8s.io/kops/util/pkg/vfs"
)

type ClusterVFS struct {
	commonVFS
}

func newClusterVFS(basePath vfs.Path) *ClusterVFS {
	c := &ClusterVFS{}
	c.init("Cluster", basePath, StoreVersion)
	defaultReadVersion := v1alpha1.SchemeGroupVersion.WithKind("Cluster")
	c.defaultReadVersion = &defaultReadVersion
	return c
}

func (c *ClusterVFS) Get(name string, options metav1.GetOptions) (*api.Cluster, error) {
	if options.ResourceVersion != "" {
		return nil, fmt.Errorf("ResourceVersion not supported in ClusterVFS::Get")
	}
	o, err := c.find(name)
	if err != nil {
		return nil, err
	}
	if o == nil {
		return nil, errors.NewNotFound(schema.GroupResource{Group: api.GroupName, Resource: "Cluster"}, name)
	}
	return o, nil
}

// Deprecated, but we need this for now..
func (c *ClusterVFS) configBase(clusterName string) (vfs.Path, error) {
	if clusterName == "" {
		return nil, fmt.Errorf("clusterName is required")
	}
	configPath := c.basePath.Join(clusterName)
	return configPath, nil
}

func (c *ClusterVFS) List(options metav1.ListOptions) (*api.ClusterList, error) {
	names, err := c.listNames()
	if err != nil {
		return nil, err
	}

	var items []api.Cluster

	for _, clusterName := range names {
		cluster, err := c.find(clusterName)
		if err != nil {
			klog.Warningf("cluster %q found in state store listing, but cannot be loaded: %v", clusterName, err)
			continue
		}

		if cluster == nil {
			klog.Warningf("cluster %q found in state store listing, but doesn't exist now", clusterName)
			continue
		}

		items = append(items, *cluster)
	}

	return &api.ClusterList{Items: items}, nil
}

func (r *ClusterVFS) Create(c *api.Cluster) (*api.Cluster, error) {
	if err := validation.ValidateCluster(c, false); err != nil {
		return nil, err
	}

	if c.ObjectMeta.CreationTimestamp.IsZero() {
		c.ObjectMeta.CreationTimestamp = metav1.NewTime(time.Now().UTC())
	}

	clusterName := c.ObjectMeta.Name
	if clusterName == "" {
		return nil, fmt.Errorf("clusterName is required")
	}

	if err := r.writeConfig(c, r.basePath.Join(clusterName, registry.PathCluster), c, vfs.WriteOptionCreate); err != nil {
		if os.IsExist(err) {
			return nil, err
		}
		return nil, fmt.Errorf("error writing Cluster %q: %v", c.ObjectMeta.Name, err)
	}

	return c, nil
}

func (r *ClusterVFS) Update(c *api.Cluster, status *api.ClusterStatus) (*api.Cluster, error) {
	clusterName := c.ObjectMeta.Name
	if clusterName == "" {
		return nil, field.Required(field.NewPath("Name"), "clusterName is required")
	}

	old, err := r.Get(clusterName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	if old == nil {
		return nil, errors.NewNotFound(schema.GroupResource{Group: api.GroupName, Resource: "Cluster"}, clusterName)
	}

	if err := validation.ValidateClusterUpdate(c, status, old).ToAggregate(); err != nil {
		return nil, err
	}

	if !apiequality.Semantic.DeepEqual(old.Spec, c.Spec) {
		c.SetGeneration(old.GetGeneration() + 1)
	}

	if err := r.writeConfig(c, r.basePath.Join(clusterName, registry.PathCluster), c, vfs.WriteOptionOnlyIfExists); err != nil {
		if os.IsNotExist(err) {
			return nil, err
		}
		return nil, fmt.Errorf("error writing Cluster: %v", err)
	}

	return c, nil
}

// List returns a slice containing all the cluster names
// It skips directories that don't look like clusters
func (r *ClusterVFS) listNames() ([]string, error) {
	paths, err := r.basePath.ReadTree()
	if err != nil {
		return nil, fmt.Errorf("error reading state store: %v", err)
	}

	var keys []string
	for _, p := range paths {
		relativePath, err := vfs.RelativePath(r.basePath, p)
		if err != nil {
			return nil, err
		}
		if !strings.HasSuffix(relativePath, "/config") {
			continue
		}
		key := strings.TrimSuffix(relativePath, "/config")
		keys = append(keys, key)
	}
	return keys, nil
}

func (r *ClusterVFS) find(clusterName string) (*api.Cluster, error) {
	if clusterName == "" {
		return nil, fmt.Errorf("clusterName is required")
	}
	configPath := r.basePath.Join(clusterName, registry.PathCluster)

	o, err := r.readConfig(configPath)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("error reading cluster configuration %q: %v", clusterName, err)
	}

	c := o.(*api.Cluster)

	if c.ObjectMeta.Name == "" {
		c.ObjectMeta.Name = clusterName
	}
	if c.ObjectMeta.Name != clusterName {
		klog.Warningf("Name of cluster does not match: %q vs %q", c.ObjectMeta.Name, clusterName)
	}

	// TODO: Split this out into real version updates / schema changes
	if c.Spec.ConfigBase == "" {
		configBase, err := r.configBase(clusterName)
		if err != nil {
			return nil, fmt.Errorf("error building ConfigBase for cluster: %v", err)
		}
		c.Spec.ConfigBase = configBase.Path()
	}

	return c, nil
}

func (r *ClusterVFS) Delete(name string, options *metav1.DeleteOptions) error {
	return fmt.Errorf("cluster Delete not implemented for vfs store")
}

func (r *ClusterVFS) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error {
	return fmt.Errorf("cluster DeleteCollection not implemented for vfs store")
}

func (r *ClusterVFS) Watch(opts metav1.ListOptions) (watch.Interface, error) {
	return nil, fmt.Errorf("cluster Watch not implemented for vfs store")
}

func (r *ClusterVFS) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *api.Cluster, err error) {
	return nil, fmt.Errorf("cluster Patch not implemented for vfs store")
}
<commit_msg>Add more debug info for when cluster path doesnt match<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vfsclientset

import (
	"fmt"
	"os"
	"strings"
	"time"

	apiequality "k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/klog"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/registry"
	"k8s.io/kops/pkg/apis/kops/v1alpha1"
	"k8s.io/kops/pkg/apis/kops/validation"
	"k8s.io/kops/util/pkg/vfs"
)

type ClusterVFS struct {
	commonVFS
}

func newClusterVFS(basePath vfs.Path) *ClusterVFS {
	c := &ClusterVFS{}
	c.init("Cluster", basePath, StoreVersion)
	defaultReadVersion := v1alpha1.SchemeGroupVersion.WithKind("Cluster")
	c.defaultReadVersion = &defaultReadVersion
	return c
}

func (c *ClusterVFS) Get(name string, options metav1.GetOptions) (*api.Cluster, error) {
	if options.ResourceVersion != "" {
		return nil, fmt.Errorf("ResourceVersion not supported in ClusterVFS::Get")
	}
	o, err := c.find(name)
	if err != nil {
		return nil, err
	}
	if o == nil {
		return nil, errors.NewNotFound(schema.GroupResource{Group: api.GroupName, Resource: "Cluster"}, name)
	}
	return o, nil
}

// Deprecated, but we need this for now..
func (c *ClusterVFS) configBase(clusterName string) (vfs.Path, error) {
	if clusterName == "" {
		return nil, fmt.Errorf("clusterName is required")
	}
	configPath := c.basePath.Join(clusterName)
	return configPath, nil
}

func (c *ClusterVFS) List(options metav1.ListOptions) (*api.ClusterList, error) {
	names, err := c.listNames()
	if err != nil {
		return nil, err
	}

	var items []api.Cluster

	for _, clusterName := range names {
		cluster, err := c.find(clusterName)
		if err != nil {
			klog.Warningf("cluster %q found in state store listing, but cannot be loaded: %v", clusterName, err)
			continue
		}

		if cluster == nil {
			klog.Warningf("cluster %q found in state store listing, but doesn't exist now", clusterName)
			continue
		}

		items = append(items, *cluster)
	}

	return &api.ClusterList{Items: items}, nil
}

func (r *ClusterVFS) Create(c *api.Cluster) (*api.Cluster, error) {
	if err := validation.ValidateCluster(c, false); err != nil {
		return nil, err
	}

	if c.ObjectMeta.CreationTimestamp.IsZero() {
		c.ObjectMeta.CreationTimestamp = metav1.NewTime(time.Now().UTC())
	}

	clusterName := c.ObjectMeta.Name
	if clusterName == "" {
		return nil, fmt.Errorf("clusterName is required")
	}

	if err := r.writeConfig(c, r.basePath.Join(clusterName, registry.PathCluster), c, vfs.WriteOptionCreate); err != nil {
		if os.IsExist(err) {
			return nil, err
		}
		return nil, fmt.Errorf("error writing Cluster %q: %v", c.ObjectMeta.Name, err)
	}

	return c, nil
}

func (r *ClusterVFS) Update(c *api.Cluster, status *api.ClusterStatus) (*api.Cluster, error) {
	clusterName := c.ObjectMeta.Name
	if clusterName == "" {
		return nil, field.Required(field.NewPath("Name"), "clusterName is required")
	}

	old, err := r.Get(clusterName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	if old == nil {
		return nil, errors.NewNotFound(schema.GroupResource{Group: api.GroupName, Resource: "Cluster"}, clusterName)
	}

	if err := validation.ValidateClusterUpdate(c, status, old).ToAggregate(); err != nil {
		return nil, err
	}

	if !apiequality.Semantic.DeepEqual(old.Spec, c.Spec) {
		c.SetGeneration(old.GetGeneration() + 1)
	}

	if err := r.writeConfig(c, r.basePath.Join(clusterName, registry.PathCluster), c, vfs.WriteOptionOnlyIfExists); err != nil {
		if os.IsNotExist(err) {
			return nil, err
		}
		return nil, fmt.Errorf("error writing Cluster: %v", err)
	}

	return c, nil
}

// List returns a slice containing all the cluster names
// It skips directories that don't look like clusters
func (r *ClusterVFS) listNames() ([]string, error) {
	paths, err := r.basePath.ReadTree()
	if err != nil {
		return nil, fmt.Errorf("error reading state store: %v", err)
	}

	var keys []string
	for _, p := range paths {
		relativePath, err := vfs.RelativePath(r.basePath, p)
		if err != nil {
			return nil, err
		}
		if !strings.HasSuffix(relativePath, "/config") {
			continue
		}
		key := strings.TrimSuffix(relativePath, "/config")
		keys = append(keys, key)
	}
	return keys, nil
}

func (r *ClusterVFS) find(clusterName string) (*api.Cluster, error) {
	if clusterName == "" {
		return nil, fmt.Errorf("clusterName is required")
	}
	configPath := r.basePath.Join(clusterName, registry.PathCluster)

	o, err := r.readConfig(configPath)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("error reading cluster configuration %q: %v", clusterName, err)
	}

	c := o.(*api.Cluster)

	if c.ObjectMeta.Name == "" {
		c.ObjectMeta.Name = clusterName
	}
	if c.ObjectMeta.Name != clusterName {
		klog.Warningf("Name of cluster does not match: actual name was %q, but cluster name was %q (using registry path %v).", c.ObjectMeta.Name, clusterName, registry.PathCluster)
	}

	// TODO: Split this out into real version updates / schema changes
	if c.Spec.ConfigBase == "" {
		configBase, err := r.configBase(clusterName)
		if err != nil {
			return nil, fmt.Errorf("error building ConfigBase for cluster: %v", err)
		}
		c.Spec.ConfigBase = configBase.Path()
	}

	return c, nil
}

func (r *ClusterVFS) Delete(name string, options *metav1.DeleteOptions) error {
	return fmt.Errorf("cluster Delete not implemented for vfs store")
}

func (r *ClusterVFS) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error {
	return fmt.Errorf("cluster DeleteCollection not implemented for vfs store")
}

func (r *ClusterVFS) Watch(opts metav1.ListOptions) (watch.Interface, error) {
	return nil, fmt.Errorf("cluster Watch not implemented for vfs store")
}

func (r *ClusterVFS) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *api.Cluster, err error) {
	return nil, fmt.Errorf("cluster Patch not implemented for vfs store")
}
<|endoftext|>


<commit_before>package dashboards

import (
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/guardian"
	"github.com/grafana/grafana/pkg/services/search"
)

// FolderService service for operating on folders
type FolderService interface {
	GetFolders(limit int) ([]*models.Folder, error)
	GetFolderById(id int64) (*models.Folder, error)
	GetFolderByUid(uid string) (*models.Folder, error)
	CreateFolder(cmd *models.CreateFolderCommand) error
	UpdateFolder(uid string, cmd *models.UpdateFolderCommand) error
	DeleteFolder(uid string) (*models.Folder, error)
}

// NewFolderService factory for creating a new folder service
var NewFolderService = func(orgId int64, user *models.SignedInUser) FolderService {
	return &dashboardServiceImpl{
		orgId: orgId,
		user:  user,
	}
}

func (dr *dashboardServiceImpl) GetFolders(limit int) ([]*models.Folder, error) {
	if limit == 0 {
		limit = 1000
	}

	searchQuery := search.Query{
		SignedInUser: dr.user,
		DashboardIds: make([]int64, 0),
		FolderIds:    make([]int64, 0),
		Limit:        limit,
		OrgId:        dr.orgId,
		Type:         "dash-folder",
		Permission:   models.PERMISSION_VIEW,
	}

	if err := bus.Dispatch(&searchQuery); err != nil {
		return nil, err
	}

	folders := make([]*models.Folder, 0)

	for _, hit := range searchQuery.Result {
		folders = append(folders, &models.Folder{
			Id:    hit.Id,
			Uid:   hit.Uid,
			Title: hit.Title,
		})
	}

	return folders, nil
}

func (dr *dashboardServiceImpl) GetFolderById(id int64) (*models.Folder, error) {
	query := models.GetDashboardQuery{OrgId: dr.orgId, Id: id}
	dashFolder, err := getFolder(query)

	if err != nil {
		return nil, toFolderError(err)
	}

	g := guardian.New(dashFolder.Id, dr.orgId, dr.user)
	if canView, err := g.CanView(); err != nil || !canView {
		if err != nil {
			return nil, toFolderError(err)
		}
		return nil, models.ErrFolderAccessDenied
	}

	return dashToFolder(dashFolder), nil
}

func (dr *dashboardServiceImpl) GetFolderByUid(uid string) (*models.Folder, error) {
	query := models.GetDashboardQuery{OrgId: dr.orgId, Uid: uid}
	dashFolder, err := getFolder(query)

	if err != nil {
		return nil, toFolderError(err)
	}

	g := guardian.New(dashFolder.Id, dr.orgId, dr.user)
	if canView, err := g.CanView(); err != nil || !canView {
		if err != nil {
			return nil, toFolderError(err)
		}
		return nil, models.ErrFolderAccessDenied
	}

	return dashToFolder(dashFolder), nil
}

func (dr *dashboardServiceImpl) CreateFolder(cmd *models.CreateFolderCommand) error {
	dashFolder := cmd.GetDashboardModel(dr.orgId, dr.user.UserId)

	dto := &SaveDashboardDTO{
		Dashboard: dashFolder,
		OrgId:     dr.orgId,
		User:      dr.user,
	}

	saveDashboardCmd, err := dr.buildSaveDashboardCommand(dto, false)
	if err != nil {
		return toFolderError(err)
	}

	err = bus.Dispatch(saveDashboardCmd)
	if err != nil {
		return toFolderError(err)
	}

	query := models.GetDashboardQuery{OrgId: dr.orgId, Id: saveDashboardCmd.Result.Id}
	dashFolder, err = getFolder(query)
	if err != nil {
		return toFolderError(err)
	}

	cmd.Result = dashToFolder(dashFolder)

	return nil
}

func (dr *dashboardServiceImpl) UpdateFolder(existingUid string, cmd *models.UpdateFolderCommand) error {
	query := models.GetDashboardQuery{OrgId: dr.orgId, Uid: existingUid}
	dashFolder, err := getFolder(query)
	if err != nil {
		return toFolderError(err)
	}

	cmd.UpdateDashboardModel(dashFolder, dr.orgId, dr.user.UserId)

	dto := &SaveDashboardDTO{
		Dashboard: dashFolder,
		OrgId:     dr.orgId,
		User:      dr.user,
		Overwrite: cmd.Overwrite,
	}

	saveDashboardCmd, err := dr.buildSaveDashboardCommand(dto, false)
	if err != nil {
		return toFolderError(err)
	}

	err = bus.Dispatch(saveDashboardCmd)
	if err != nil {
		return toFolderError(err)
	}

	query = models.GetDashboardQuery{OrgId: dr.orgId, Id: saveDashboardCmd.Result.Id}
	dashFolder, err = getFolder(query)
	if err != nil {
		return toFolderError(err)
	}

	cmd.Result = dashToFolder(dashFolder)

	return nil
}

func (dr *dashboardServiceImpl) DeleteFolder(uid string) (*models.Folder, error) {
	query := models.GetDashboardQuery{OrgId: dr.orgId, Uid: uid}
	dashFolder, err := getFolder(query)
	if err != nil {
		return nil, toFolderError(err)
	}

	guardian := guardian.New(dashFolder.Id, dr.orgId, dr.user)
	if canSave, err := guardian.CanSave(); err != nil || !canSave {
		if err != nil {
			return nil, toFolderError(err)
		}
		return nil, models.ErrFolderAccessDenied
	}

	deleteCmd := models.DeleteDashboardCommand{OrgId: dr.orgId, Id: dashFolder.Id}
	if err := bus.Dispatch(&deleteCmd); err != nil {
		return nil, toFolderError(err)
	}

	return dashToFolder(dashFolder), nil
}

func getFolder(query models.GetDashboardQuery) (*models.Dashboard, error) {
	if err := bus.Dispatch(&query); err != nil {
		return nil, toFolderError(err)
	}

	if !query.Result.IsFolder {
		return nil, models.ErrFolderNotFound
	}

	return query.Result, nil
}

func dashToFolder(dash *models.Dashboard) *models.Folder {
	return &models.Folder{
		Id:        dash.Id,
		Uid:       dash.Uid,
		Title:     dash.Title,
		HasAcl:    dash.HasAcl,
		Url:       dash.GetUrl(),
		Version:   dash.Version,
		Created:   dash.Created,
		CreatedBy: dash.CreatedBy,
		Updated:   dash.Updated,
		UpdatedBy: dash.UpdatedBy,
	}
}

func toFolderError(err error) error {
	if err == models.ErrDashboardTitleEmpty {
		return models.ErrFolderTitleEmpty
	}

	if err == models.ErrDashboardUpdateAccessDenied {
		return models.ErrFolderAccessDenied
	}

	if err == models.ErrDashbloardWithSameNameInFolderExists {
		return models.ErrFolderSameNameExists
	}

	if err == models.ErrDashboardWithSameUIDExists {
		return models.ErrFolderWithSameUIDExists
	}

	if err == models.ErrDashboardVersionMismatch {
		return models.ErrFolderVersionMismatch
	}

	if err == models.ErrDashboardNotFound {
		return models.ErrFolderNotFound
	}

	if err == models.ErrDashboardFailedGenerateUniqueUid {
		err = models.ErrFolderFailedGenerateUniqueUid
	}

	return err
}
<commit_msg>folders: fix typo<commit_after>package dashboards

import (
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/guardian"
	"github.com/grafana/grafana/pkg/services/search"
)

// FolderService service for operating on folders
type FolderService interface {
	GetFolders(limit int) ([]*models.Folder, error)
	GetFolderById(id int64) (*models.Folder, error)
	GetFolderByUid(uid string) (*models.Folder, error)
	CreateFolder(cmd *models.CreateFolderCommand) error
	UpdateFolder(uid string, cmd *models.UpdateFolderCommand) error
	DeleteFolder(uid string) (*models.Folder, error)
}

// NewFolderService factory for creating a new folder service
var NewFolderService = func(orgId int64, user *models.SignedInUser) FolderService {
	return &dashboardServiceImpl{
		orgId: orgId,
		user:  user,
	}
}

func (dr *dashboardServiceImpl) GetFolders(limit int) ([]*models.Folder, error) {
	if limit == 0 {
		limit = 1000
	}

	searchQuery := search.Query{
		SignedInUser: dr.user,
		DashboardIds: make([]int64, 0),
		FolderIds:    make([]int64, 0),
		Limit:        limit,
		OrgId:        dr.orgId,
		Type:         "dash-folder",
		Permission:   models.PERMISSION_VIEW,
	}

	if err := bus.Dispatch(&searchQuery); err != nil {
		return nil, err
	}

	folders := make([]*models.Folder, 0)

	for _, hit := range searchQuery.Result {
		folders = append(folders, &models.Folder{
			Id:    hit.Id,
			Uid:   hit.Uid,
			Title: hit.Title,
		})
	}

	return folders, nil
}

func (dr *dashboardServiceImpl) GetFolderById(id int64) (*models.Folder, error) {
	query := models.GetDashboardQuery{OrgId: dr.orgId, Id: id}
	dashFolder, err := getFolder(query)

	if err != nil {
		return nil, toFolderError(err)
	}

	g := guardian.New(dashFolder.Id, dr.orgId, dr.user)
	if canView, err := g.CanView(); err != nil || !canView {
		if err != nil {
			return nil, toFolderError(err)
		}
		return nil, models.ErrFolderAccessDenied
	}

	return dashToFolder(dashFolder), nil
}

func (dr *dashboardServiceImpl) GetFolderByUid(uid string) (*models.Folder, error) {
	query := models.GetDashboardQuery{OrgId: dr.orgId, Uid: uid}
	dashFolder, err := getFolder(query)

	if err != nil {
		return nil, toFolderError(err)
	}

	g := guardian.New(dashFolder.Id, dr.orgId, dr.user)
	if canView, err := g.CanView(); err != nil || !canView {
		if err != nil {
			return nil, toFolderError(err)
		}
		return nil, models.ErrFolderAccessDenied
	}

	return dashToFolder(dashFolder), nil
}

func (dr *dashboardServiceImpl) CreateFolder(cmd *models.CreateFolderCommand) error {
	dashFolder := cmd.GetDashboardModel(dr.orgId, dr.user.UserId)

	dto := &SaveDashboardDTO{
		Dashboard: dashFolder,
		OrgId:     dr.orgId,
		User:      dr.user,
	}

	saveDashboardCmd, err := dr.buildSaveDashboardCommand(dto, false)
	if err != nil {
		return toFolderError(err)
	}

	err = bus.Dispatch(saveDashboardCmd)
	if err != nil {
		return toFolderError(err)
	}

	query := models.GetDashboardQuery{OrgId: dr.orgId, Id: saveDashboardCmd.Result.Id}
	dashFolder, err = getFolder(query)
	if err != nil {
		return toFolderError(err)
	}

	cmd.Result = dashToFolder(dashFolder)

	return nil
}

func (dr *dashboardServiceImpl) UpdateFolder(existingUid string, cmd *models.UpdateFolderCommand) error {
	query := models.GetDashboardQuery{OrgId: dr.orgId, Uid: existingUid}
	dashFolder, err := getFolder(query)
	if err != nil {
		return toFolderError(err)
	}

	cmd.UpdateDashboardModel(dashFolder, dr.orgId, dr.user.UserId)

	dto := &SaveDashboardDTO{
		Dashboard: dashFolder,
		OrgId:     dr.orgId,
		User:      dr.user,
		Overwrite: cmd.Overwrite,
	}

	saveDashboardCmd, err := dr.buildSaveDashboardCommand(dto, false)
	if err != nil {
		return toFolderError(err)
	}

	err = bus.Dispatch(saveDashboardCmd)
	if err != nil {
		return toFolderError(err)
	}

	query = models.GetDashboardQuery{OrgId: dr.orgId, Id: saveDashboardCmd.Result.Id}
	dashFolder, err = getFolder(query)
	if err != nil {
		return toFolderError(err)
	}

	cmd.Result = dashToFolder(dashFolder)

	return nil
}

func (dr *dashboardServiceImpl) DeleteFolder(uid string) (*models.Folder, error) {
	query := models.GetDashboardQuery{OrgId: dr.orgId, Uid: uid}
	dashFolder, err := getFolder(query)
	if err != nil {
		return nil, toFolderError(err)
	}

	guardian := guardian.New(dashFolder.Id, dr.orgId, dr.user)
	if canSave, err := guardian.CanSave(); err != nil || !canSave {
		if err != nil {
			return nil, toFolderError(err)
		}
		return nil, models.ErrFolderAccessDenied
	}

	deleteCmd := models.DeleteDashboardCommand{OrgId: dr.orgId, Id: dashFolder.Id}
	if err := bus.Dispatch(&deleteCmd); err != nil {
		return nil, toFolderError(err)
	}

	return dashToFolder(dashFolder), nil
}

func getFolder(query models.GetDashboardQuery) (*models.Dashboard, error) {
	if err := bus.Dispatch(&query); err != nil {
		return nil, toFolderError(err)
	}

	if !query.Result.IsFolder {
		return nil, models.ErrFolderNotFound
	}

	return query.Result, nil
}

func dashToFolder(dash *models.Dashboard) *models.Folder {
	return &models.Folder{
		Id:        dash.Id,
		Uid:       dash.Uid,
		Title:     dash.Title,
		HasAcl:    dash.HasAcl,
		Url:       dash.GetUrl(),
		Version:   dash.Version,
		Created:   dash.Created,
		CreatedBy: dash.CreatedBy,
		Updated:   dash.Updated,
		UpdatedBy: dash.UpdatedBy,
	}
}

func toFolderError(err error) error {
	if err == models.ErrDashboardTitleEmpty {
		return models.ErrFolderTitleEmpty
	}

	if err == models.ErrDashboardUpdateAccessDenied {
		return models.ErrFolderAccessDenied
	}

	if err == models.ErrDashboardWithSameNameInFolderExists {
		return models.ErrFolderSameNameExists
	}

	if err == models.ErrDashboardWithSameUIDExists {
		return models.ErrFolderWithSameUIDExists
	}

	if err == models.ErrDashboardVersionMismatch {
		return models.ErrFolderVersionMismatch
	}

	if err == models.ErrDashboardNotFound {
		return models.ErrFolderNotFound
	}

	if err == models.ErrDashboardFailedGenerateUniqueUid {
		err = models.ErrFolderFailedGenerateUniqueUid
	}

	return err
}
<|endoftext|>


<commit_before>package storageservice

import (
	"encoding/xml"
	"errors"
	"fmt"
	"strings"

	"github.com/MSOpenTech/azure-sdk-for-go/management"
)

const (
	azureStorageServiceListURL         = "services/storageservices"
	azureStorageServiceURL             = "services/storageservices/%s"
	azureStorageAccountAvailabilityURL = "services/storageservices/operations/isavailable/%s"

	azureXmlns = "http://schemas.microsoft.com/windowsazure"

	errBlobEndpointNotFound = "Blob endpoint was not found in storage serice %s"
	errParamNotSpecified    = "Parameter %s is not specified."
)

//NewClient is used to instantiate a new StorageServiceClient from an Azure client
func NewClient(self management.Client) StorageServiceClient {
	return StorageServiceClient{client: self}
}

func (self StorageServiceClient) GetStorageServiceList() (*StorageServiceList, error) {
	storageServiceList := new(StorageServiceList)

	response, err := self.client.SendAzureGetRequest(azureStorageServiceListURL)
	if err != nil {
		return nil, err
	}

	err = xml.Unmarshal(response, storageServiceList)
	if err != nil {
		return storageServiceList, err
	}

	return storageServiceList, nil
}

func (self StorageServiceClient) GetStorageServiceByName(serviceName string) (*StorageService, error) {
	if serviceName == "" {
		return nil, fmt.Errorf(errParamNotSpecified, "serviceName")
	}

	storageService := new(StorageService)
	requestURL := fmt.Sprintf(azureStorageServiceURL, serviceName)
	response, err := self.client.SendAzureGetRequest(requestURL)
	if err != nil {
		return nil, err
	}

	err = xml.Unmarshal(response, storageService)
	if err != nil {
		return nil, err
	}

	return storageService, nil
}

func (self StorageServiceClient) GetStorageServiceByLocation(location string) (*StorageService, error) {
	if location == "" {
		return nil, fmt.Errorf(errParamNotSpecified, "location")
	}

	storageService := new(StorageService)
	storageServiceList, err := self.GetStorageServiceList()
	if err != nil {
		return storageService, err
	}

	for _, storageService := range storageServiceList.StorageServices {
		if storageService.StorageServiceProperties.Location != location {
			continue
		}

		return &storageService, nil
	}

	return nil, nil
}

func (self StorageServiceClient) GetStorageServiceKeys(serviceName string) (GetStorageServiceKeysResponse, error) {
	if serviceName == "" {
		return GetStorageServiceKeysResponse{}, fmt.Errorf(errParamNotSpecified, "serviceName")
	}

	requestURL := fmt.Sprintf(azureStorageServiceURL, serviceName)
	data, err := self.client.SendAzureGetRequest(requestURL)
	if err != nil {
		return GetStorageServiceKeysResponse{}, err
	}

	response := GetStorageServiceKeysResponse{}
	err = xml.Unmarshal(data, &response)

	return response, err
}

func (self StorageServiceClient) CreateAsync(parameters StorageAccountCreateParameters) (management.OperationId, error) {
	data, err := xml.Marshal(CreateStorageServiceInput{
		StorageAccountCreateParameters: parameters})
	if err != nil {
		return "", err
	}

	return self.client.SendAzurePostRequest(azureStorageServiceListURL, data)
}

func (self StorageServiceClient) Create(parameters StorageAccountCreateParameters) (*StorageService, error) {
	operationId, err := self.CreateAsync(parameters)
	if err != nil {
		return nil, err
	}

	err = self.client.WaitAsyncOperation(operationId)
	if err != nil {
		return nil, err
	}

	storageService, err := self.GetStorageServiceByName(parameters.ServiceName)
	if err != nil {
		return nil, err
	}

	return storageService, nil
}

func (self StorageServiceClient) GetBlobEndpoint(storageService *StorageService) (string, error) {
	for _, endpoint := range storageService.StorageServiceProperties.Endpoints {
		if !strings.Contains(endpoint, ".blob.core") {
			continue
		}

		return endpoint, nil
	}

	return "", errors.New(fmt.Sprintf(errBlobEndpointNotFound, storageService.ServiceName))
}

// The Check Storage Account Name Availability operation checks to see if the specified storage account name is available, or if it has already been taken.
// See https://msdn.microsoft.com/en-us/library/azure/jj154125.aspx
func (self StorageServiceClient) IsAvailable(name string) (bool, string, error) {
	if name == "" {
		return false, "", fmt.Errorf(errParamNotSpecified, "name")
	}

	requestURL := fmt.Sprintf(azureStorageAccountAvailabilityURL, name)
	response, err := self.client.SendAzureGetRequest(requestURL)
	if err != nil {
		return false, "", err
	}

	availabilityResponse := new(AvailabilityResponse)
	err = xml.Unmarshal(response, availabilityResponse)
	if err != nil {
		return false, "", err
	}

	return availabilityResponse.Result, availabilityResponse.Reason, nil
}
<commit_msg>Fix url for GetStorageServiceKeys<commit_after>package storageservice

import (
	"encoding/xml"
	"errors"
	"fmt"
	"strings"

	"github.com/MSOpenTech/azure-sdk-for-go/management"
)

const (
	azureStorageServiceListURL         = "services/storageservices"
	azureStorageServiceURL             = "services/storageservices/%s"
	azureStorageServiceKeysURL         = "services/storageservices/%s/keys"
	azureStorageAccountAvailabilityURL = "services/storageservices/operations/isavailable/%s"

	azureXmlns = "http://schemas.microsoft.com/windowsazure"

	errBlobEndpointNotFound = "Blob endpoint was not found in storage serice %s"
	errParamNotSpecified    = "Parameter %s is not specified."
)

//NewClient is used to instantiate a new StorageServiceClient from an Azure client
func NewClient(self management.Client) StorageServiceClient {
	return StorageServiceClient{client: self}
}

func (self StorageServiceClient) GetStorageServiceList() (*StorageServiceList, error) {
	storageServiceList := new(StorageServiceList)

	response, err := self.client.SendAzureGetRequest(azureStorageServiceListURL)
	if err != nil {
		return nil, err
	}

	err = xml.Unmarshal(response, storageServiceList)
	if err != nil {
		return storageServiceList, err
	}

	return storageServiceList, nil
}

func (self StorageServiceClient) GetStorageServiceByName(serviceName string) (*StorageService, error) {
	if serviceName == "" {
		return nil, fmt.Errorf(errParamNotSpecified, "serviceName")
	}

	storageService := new(StorageService)
	requestURL := fmt.Sprintf(azureStorageServiceURL, serviceName)
	response, err := self.client.SendAzureGetRequest(requestURL)
	if err != nil {
		return nil, err
	}

	err = xml.Unmarshal(response, storageService)
	if err != nil {
		return nil, err
	}

	return storageService, nil
}

func (self StorageServiceClient) GetStorageServiceByLocation(location string) (*StorageService, error) {
	if location == "" {
		return nil, fmt.Errorf(errParamNotSpecified, "location")
	}

	storageService := new(StorageService)
	storageServiceList, err := self.GetStorageServiceList()
	if err != nil {
		return storageService, err
	}

	for _, storageService := range storageServiceList.StorageServices {
		if storageService.StorageServiceProperties.Location != location {
			continue
		}

		return &storageService, nil
	}

	return nil, nil
}

func (self StorageServiceClient) GetStorageServiceKeys(serviceName string) (GetStorageServiceKeysResponse, error) {
	if serviceName == "" {
		return GetStorageServiceKeysResponse{}, fmt.Errorf(errParamNotSpecified, "serviceName")
	}

	requestURL := fmt.Sprintf(azureStorageServiceKeysURL, serviceName)
	data, err := self.client.SendAzureGetRequest(requestURL)
	if err != nil {
		return GetStorageServiceKeysResponse{}, err
	}

	response := GetStorageServiceKeysResponse{}
	err = xml.Unmarshal(data, &response)

	return response, err
}

func (self StorageServiceClient) CreateAsync(parameters StorageAccountCreateParameters) (management.OperationId, error) {
	data, err := xml.Marshal(CreateStorageServiceInput{
		StorageAccountCreateParameters: parameters})
	if err != nil {
		return "", err
	}

	return self.client.SendAzurePostRequest(azureStorageServiceListURL, data)
}

func (self StorageServiceClient) Create(parameters StorageAccountCreateParameters) (*StorageService, error) {
	operationId, err := self.CreateAsync(parameters)
	if err != nil {
		return nil, err
	}

	err = self.client.WaitAsyncOperation(operationId)
	if err != nil {
		return nil, err
	}

	storageService, err := self.GetStorageServiceByName(parameters.ServiceName)
	if err != nil {
		return nil, err
	}

	return storageService, nil
}

func (self StorageServiceClient) GetBlobEndpoint(storageService *StorageService) (string, error) {
	for _, endpoint := range storageService.StorageServiceProperties.Endpoints {
		if !strings.Contains(endpoint, ".blob.core") {
			continue
		}

		return endpoint, nil
	}

	return "", errors.New(fmt.Sprintf(errBlobEndpointNotFound, storageService.ServiceName))
}

// The Check Storage Account Name Availability operation checks to see if the specified storage account name is available, or if it has already been taken.
// See https://msdn.microsoft.com/en-us/library/azure/jj154125.aspx
func (self StorageServiceClient) IsAvailable(name string) (bool, string, error) {
	if name == "" {
		return false, "", fmt.Errorf(errParamNotSpecified, "name")
	}

	requestURL := fmt.Sprintf(azureStorageAccountAvailabilityURL, name)
	response, err := self.client.SendAzureGetRequest(requestURL)
	if err != nil {
		return false, "", err
	}

	availabilityResponse := new(AvailabilityResponse)
	err = xml.Unmarshal(response, availabilityResponse)
	if err != nil {
		return false, "", err
	}

	return availabilityResponse.Result, availabilityResponse.Reason, nil
}
<|endoftext|>