<commit_before>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import "testing"

func TestStrIsTrue(t *testing.T) {

	if !StrIsTrue("true") {
		t.Errorf("TestStrIsTrue is broken - expecting true - string param: true")
	}

	if !StrIsTrue("yes") {
		t.Errorf("TestStrIsTrue is broken - expecting true - string param: yes")
	}

	if !StrIsTrue("y") {
		t.Errorf("TestStrIsTrue is broken - expecting true - string param: y")
	}

	if !StrIsTrue("1") {
		t.Errorf("TestStrIsTrue is broken - expecting true - string param: 1")
	}

	if StrIsTrue("g") {
		t.Errorf("TestStrIsTrue is broken - expecting false - string param: g")
	}

	if StrIsTrue("false") {
		t.Errorf("TestStrIsTrue is broken - expecting false - string param: false")
	}

	if StrIsTrue("no") {
		t.Errorf("TestStrIsTrue is broken - expecting false - string param: no")
	}

	if StrIsTrue("n") {
		t.Errorf("TestStrIsTrue is broken - expecting false - string param: n")
	}

	if StrIsTrue("0") {
		t.Errorf("TestStrIsTrue is broken - expecting false - string param: 0")
	}
}

func TestStrIsFalse(t *testing.T) {

	if StrIsFalse("true") {
		t.Errorf("TestStrIsFalse is broken - expecting false - string param: true")
	}

	if StrIsFalse("yes") {
		t.Errorf("TestStrIsFalse is broken - expecting false - string param: yes")
	}

	if StrIsFalse("y") {
		t.Errorf("TestStrIsFalse is broken - expecting false - string param: y")
	}

	if StrIsFalse("1") {
		t.Errorf("TestStrIsFalse is broken - expecting false - string param: 1")
	}

	if StrIsFalse("g") {
		t.Errorf("TestStrIsFalse is broken - expecting false - string param: g")
	}

	if !StrIsFalse("false") {
		t.Errorf("TestStrIsFalse is broken - expecting true - string param: false")
	}

	if !StrIsFalse("no") {
		t.Errorf("TestStrIsFalse is broken - expecting true - string param: no")
	}

	if !StrIsFalse("n") {
		t.Errorf("TestStrIsFalse is broken - expecting true - string param: n")
	}

	if !StrIsFalse("0") {
		t.Errorf("TestStrIsFalse is broken - expecting true - string param: 0")
	}
}

<commit_msg>layout<commit_after>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import "testing"

func TestStrIsTrue(t *testing.T) {

	if !StrIsTrue("true") { t.Errorf("TestStrIsTrue is broken - expecting true - string param: true") }

	if !StrIsTrue("yes") {
		t.Errorf("TestStrIsTrue is broken - expecting true - string param: yes")
	}

	if !StrIsTrue("y") {
		t.Errorf("TestStrIsTrue is broken - expecting true - string param: y")
	}

	if !StrIsTrue("1") {
		t.Errorf("TestStrIsTrue is broken - expecting true - string param: 1")
	}

	if StrIsTrue("g") {
		t.Errorf("TestStrIsTrue is broken - expecting false - string param: g")
	}

	if StrIsTrue("false") {
		t.Errorf("TestStrIsTrue is broken - expecting false - string param: false")
	}

	if StrIsTrue("no") {
		t.Errorf("TestStrIsTrue is broken - expecting false - string param: no")
	}

	if StrIsTrue("n") {
		t.Errorf("TestStrIsTrue is broken - expecting false - string param: n")
	}

	if StrIsTrue("0") {
		t.Errorf("TestStrIsTrue is broken - expecting false - string param: 0")
	}
}

func TestStrIsFalse(t *testing.T) {

	if StrIsFalse("true") {
		t.Errorf("TestStrIsFalse is broken - expecting false - string param: true")
	}

	if StrIsFalse("yes") {
		t.Errorf("TestStrIsFalse is broken - expecting false - string param: yes")
	}

	if StrIsFalse("y") {
		t.Errorf("TestStrIsFalse is broken - expecting false - string param: y")
	}

	if StrIsFalse("1") {
		t.Errorf("TestStrIsFalse is broken - expecting false - string param: 1")
	}

	if StrIsFalse("g") {
		t.Errorf("TestStrIsFalse is broken - expecting false - string param: g")
	}

	if !StrIsFalse("false") {
		t.Errorf("TestStrIsFalse is broken - expecting true - string param: false")
	}

	if !StrIsFalse("no") {
		t.Errorf("TestStrIsFalse is broken - expecting true - string param: no")
	}

	if !StrIsFalse("n") {
		t.Errorf("TestStrIsFalse is broken - expecting true - string param: n")
	}

	if !StrIsFalse("0") {
		t.Errorf("TestStrIsFalse is broken - expecting true - string param: 0")
	}
}

<|endoftext|>


<commit_before>package bosh

import (
	"github.com/EngineerBetter/concourse-up/bosh/internal/boshenv"
	"github.com/EngineerBetter/concourse-up/bosh/internal/gcp"
)

// Deploy deploys a new Bosh director or converges an existing deployment
// Returns new contents of bosh state file
func (client *GCPClient) Deploy(state, creds []byte, detach bool) (newState, newCreds []byte, err error) {
	bosh, err := boshenv.New(boshenv.DownloadBOSH())
	if err != nil {
		return state, creds, err
	}

	state, creds, err = client.createEnv(bosh, state, creds, "")
	if err != nil {
		return state, creds, err
	}

	if err = client.updateCloudConfig(bosh); err != nil {
		return state, creds, err
	}
	if err = client.uploadConcourseStemcell(bosh); err != nil {
		return state, creds, err
	}
	if err = client.createDefaultDatabases(); err != nil {
		return state, creds, err
	}

	creds, err = client.deployConcourse(creds, detach)
	if err != nil {
		return state, creds, err
	}

	return state, creds, err
}

// CreateEnv exposes bosh create-env functionality
func (client *GCPClient) CreateEnv(state, creds []byte, customOps string) (newState, newCreds []byte, err error) {
	bosh, err := boshenv.New(boshenv.DownloadBOSH())
	if err != nil {
		return state, creds, err
	}
	return client.createEnv(bosh, state, creds, customOps)
}

// Recreate exposes BOSH recreate
func (client *GCPClient) Recreate() error {
	bosh, err := boshenv.New(boshenv.DownloadBOSH())
	if err != nil {
		return err
	}
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return err
	}
	return bosh.UploadConcourseStemcell(gcp.Environment{
		ExternalIP: directorPublicIP,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)
}

func (client *GCPClient) createEnv(bosh *boshenv.BOSHCLI, state, creds []byte, customOps string) (newState, newCreds []byte, err error) {
	tags, err := splitTags(client.config.Tags)
	if err != nil {
		return state, creds, err
	}
	tags["concourse-up-project"] = client.config.Project
	tags["concourse-up-component"] = "concourse"
	//TODO(px): pull up this so that we use aws.Store
	store := temporaryStore{
		"vars.yaml":  creds,
		"state.json": state,
	}

	network, err1 := client.metadata.Get("Network")
	if err1 != nil {
		return state, creds, err1
	}
	publicSubnetwork, err1 := client.metadata.Get("PublicSubnetworkName")
	if err1 != nil {
		return state, creds, err1
	}
	privateSubnetwork, err1 := client.metadata.Get("PrivateSubnetworkName")
	if err1 != nil {
		return state, creds, err1
	}
	directorPublicIP, err1 := client.metadata.Get("DirectorPublicIP")
	if err1 != nil {
		return state, creds, err1
	}
	project, err1 := client.provider.Attr("project")
	if err1 != nil {
		return state, creds, err1
	}
	// credentialsPath := "~/Downloads/concourse-up-6f707de4d7bd.json"
	credentialsPath, err1 := client.provider.Attr("credentials_path")
	if err1 != nil {
		return state, creds, err1
	}
	err1 = bosh.CreateEnv(store, gcp.Environment{
		InternalCIDR:       "10.0.0.0/24",
		InternalGW:         "10.0.0.1",
		InternalIP:         "10.0.0.6",
		DirectorName:       "bosh",
		Zone:               client.provider.Zone(""),
		Network:            network,
		PublicSubnetwork:   publicSubnetwork,
		PrivateSubnetwork:  privateSubnetwork,
		Tags:               "[internal]",
		ProjectID:          project,
		GcpCredentialsJSON: credentialsPath,
		ExternalIP:         directorPublicIP,
		Spot:               client.config.Spot,
		PublicKey:          client.config.PublicKey,
		CustomOperations:   customOps,
	}, client.config.DirectorPassword, client.config.DirectorCert, client.config.DirectorKey, client.config.DirectorCACert, tags)
	if err1 != nil {
		return store["state.json"], store["vars.yaml"], err1
	}
	return store["state.json"], store["vars.yaml"], err
}

// Locks implements locks for GCP client
func (client *GCPClient) Locks() ([]byte, error) {
	bosh, err := boshenv.New(boshenv.DownloadBOSH())
	if err != nil {
		return []byte{}, err
	}
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return nil, err
	}
	return bosh.Locks(gcp.Environment{
		ExternalIP: directorPublicIP,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)

}

func (client *GCPClient) updateCloudConfig(bosh *boshenv.BOSHCLI) error {

	privateSubnetwork, err := client.metadata.Get("PrivateSubnetworkName")
	if err != nil {
		return err
	}
	publicSubnetwork, err := client.metadata.Get("PublicSubnetworkName")
	if err != nil {
		return err
	}
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return err
	}
	network, err := client.metadata.Get("Network")
	if err != nil {
		return err
	}
	zone := client.provider.Zone("")
	return bosh.UpdateCloudConfig(gcp.Environment{
		Spot:              client.config.Spot,
		PublicSubnetwork:  publicSubnetwork,
		PrivateSubnetwork: privateSubnetwork,
		Zone:              zone,
		Network:           network,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)
}
func (client *GCPClient) uploadConcourseStemcell(bosh *boshenv.BOSHCLI) error {
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return err
	}
	return bosh.UploadConcourseStemcell(gcp.Environment{
		ExternalIP: directorPublicIP,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)
}

func (client *GCPClient) saveStateFile(bytes []byte) (string, error) {
	if bytes == nil {
		return client.director.PathInWorkingDir(StateFilename), nil
	}

	return client.director.SaveFileToWorkingDir(StateFilename, bytes)
}

func (client *GCPClient) saveCredsFile(bytes []byte) (string, error) {
	if bytes == nil {
		return client.director.PathInWorkingDir(CredsFilename), nil
	}

	return client.director.SaveFileToWorkingDir(CredsFilename, bytes)
}
<commit_msg>actually call bosh recreate when calling recreate<commit_after>package bosh

import (
	"github.com/EngineerBetter/concourse-up/bosh/internal/boshenv"
	"github.com/EngineerBetter/concourse-up/bosh/internal/gcp"
)

// Deploy deploys a new Bosh director or converges an existing deployment
// Returns new contents of bosh state file
func (client *GCPClient) Deploy(state, creds []byte, detach bool) (newState, newCreds []byte, err error) {
	bosh, err := boshenv.New(boshenv.DownloadBOSH())
	if err != nil {
		return state, creds, err
	}

	state, creds, err = client.createEnv(bosh, state, creds, "")
	if err != nil {
		return state, creds, err
	}

	if err = client.updateCloudConfig(bosh); err != nil {
		return state, creds, err
	}
	if err = client.uploadConcourseStemcell(bosh); err != nil {
		return state, creds, err
	}
	if err = client.createDefaultDatabases(); err != nil {
		return state, creds, err
	}

	creds, err = client.deployConcourse(creds, detach)
	if err != nil {
		return state, creds, err
	}

	return state, creds, err
}

// CreateEnv exposes bosh create-env functionality
func (client *GCPClient) CreateEnv(state, creds []byte, customOps string) (newState, newCreds []byte, err error) {
	bosh, err := boshenv.New(boshenv.DownloadBOSH())
	if err != nil {
		return state, creds, err
	}
	return client.createEnv(bosh, state, creds, customOps)
}

// Recreate exposes BOSH recreate
func (client *GCPClient) Recreate() error {
	bosh, err := boshenv.New(boshenv.DownloadBOSH())
	if err != nil {
		return err
	}
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return err
	}
	return bosh.Recreate(gcp.Environment{
		ExternalIP: directorPublicIP,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)
}

func (client *GCPClient) createEnv(bosh *boshenv.BOSHCLI, state, creds []byte, customOps string) (newState, newCreds []byte, err error) {
	tags, err := splitTags(client.config.Tags)
	if err != nil {
		return state, creds, err
	}
	tags["concourse-up-project"] = client.config.Project
	tags["concourse-up-component"] = "concourse"
	//TODO(px): pull up this so that we use aws.Store
	store := temporaryStore{
		"vars.yaml":  creds,
		"state.json": state,
	}

	network, err1 := client.metadata.Get("Network")
	if err1 != nil {
		return state, creds, err1
	}
	publicSubnetwork, err1 := client.metadata.Get("PublicSubnetworkName")
	if err1 != nil {
		return state, creds, err1
	}
	privateSubnetwork, err1 := client.metadata.Get("PrivateSubnetworkName")
	if err1 != nil {
		return state, creds, err1
	}
	directorPublicIP, err1 := client.metadata.Get("DirectorPublicIP")
	if err1 != nil {
		return state, creds, err1
	}
	project, err1 := client.provider.Attr("project")
	if err1 != nil {
		return state, creds, err1
	}
	// credentialsPath := "~/Downloads/concourse-up-6f707de4d7bd.json"
	credentialsPath, err1 := client.provider.Attr("credentials_path")
	if err1 != nil {
		return state, creds, err1
	}
	err1 = bosh.CreateEnv(store, gcp.Environment{
		InternalCIDR:       "10.0.0.0/24",
		InternalGW:         "10.0.0.1",
		InternalIP:         "10.0.0.6",
		DirectorName:       "bosh",
		Zone:               client.provider.Zone(""),
		Network:            network,
		PublicSubnetwork:   publicSubnetwork,
		PrivateSubnetwork:  privateSubnetwork,
		Tags:               "[internal]",
		ProjectID:          project,
		GcpCredentialsJSON: credentialsPath,
		ExternalIP:         directorPublicIP,
		Spot:               client.config.Spot,
		PublicKey:          client.config.PublicKey,
		CustomOperations:   customOps,
	}, client.config.DirectorPassword, client.config.DirectorCert, client.config.DirectorKey, client.config.DirectorCACert, tags)
	if err1 != nil {
		return store["state.json"], store["vars.yaml"], err1
	}
	return store["state.json"], store["vars.yaml"], err
}

// Locks implements locks for GCP client
func (client *GCPClient) Locks() ([]byte, error) {
	bosh, err := boshenv.New(boshenv.DownloadBOSH())
	if err != nil {
		return []byte{}, err
	}
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return nil, err
	}
	return bosh.Locks(gcp.Environment{
		ExternalIP: directorPublicIP,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)

}

func (client *GCPClient) updateCloudConfig(bosh *boshenv.BOSHCLI) error {

	privateSubnetwork, err := client.metadata.Get("PrivateSubnetworkName")
	if err != nil {
		return err
	}
	publicSubnetwork, err := client.metadata.Get("PublicSubnetworkName")
	if err != nil {
		return err
	}
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return err
	}
	network, err := client.metadata.Get("Network")
	if err != nil {
		return err
	}
	zone := client.provider.Zone("")
	return bosh.UpdateCloudConfig(gcp.Environment{
		Spot:              client.config.Spot,
		PublicSubnetwork:  publicSubnetwork,
		PrivateSubnetwork: privateSubnetwork,
		Zone:              zone,
		Network:           network,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)
}
func (client *GCPClient) uploadConcourseStemcell(bosh *boshenv.BOSHCLI) error {
	directorPublicIP, err := client.metadata.Get("DirectorPublicIP")
	if err != nil {
		return err
	}
	return bosh.UploadConcourseStemcell(gcp.Environment{
		ExternalIP: directorPublicIP,
	}, directorPublicIP, client.config.DirectorPassword, client.config.DirectorCACert)
}

func (client *GCPClient) saveStateFile(bytes []byte) (string, error) {
	if bytes == nil {
		return client.director.PathInWorkingDir(StateFilename), nil
	}

	return client.director.SaveFileToWorkingDir(StateFilename, bytes)
}

func (client *GCPClient) saveCredsFile(bytes []byte) (string, error) {
	if bytes == nil {
		return client.director.PathInWorkingDir(CredsFilename), nil
	}

	return client.director.SaveFileToWorkingDir(CredsFilename, bytes)
}
<|endoftext|>


<commit_before>package store

import (
	"fmt"

	"github.com/go-redis/redis/v7"
	log "github.com/sirupsen/logrus"
)

type Store struct {
	redis *redis.Client
}

func NewStore() *Store {
	client := redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "",
		DB:       0,
	})

	_, err := client.Ping().Result()
	if err != nil {
		log.Fatal(err)
	}
	log.Info("[store] connection to redis established")

	return &Store{
		redis: client,
	}
}

func (s *Store) UpdateMsgps(channelID string, msgps int64) {
	s.redis.ZAdd("channel:msgps", &redis.Z{Score: float64(msgps), Member: channelID})
}

func (s *Store) GetMsgps(channelID string) float64 {
	score, err := s.redis.ZScore("channel:msgps", channelID).Result()
	if err != nil {
		return 0
	}

	return score
}

func (s *Store) GetMsgpsScores() []redis.Z {
	scores, err := s.redis.ZRevRangeWithScores("channel:msgps", 0, 9).Result()
	if err != nil {
		return []redis.Z{}
	}

	return scores
}

func (s *Store) AddChannels(channelIDs ...string) {
	for _, id := range channelIDs {
		_, err := s.redis.HSet("channels", id, "1").Result()
		if err != nil {
			log.Error(err)
			continue
		}
	}
	log.Infof("[store] added %v", channelIDs)
}

type cloudWord struct {
	Word string
	Add  int64
}

func (s *Store) AddToWordcloud(words ...string) {
	for _, word := range words {
		_, err := s.redis.ZIncrBy("wordcloud", 1, word).Result()
		if err != nil {
			log.Error(err)
			continue
		}
	}
}

func (s *Store) GetEntireWordcloud() map[string]float64 {
	words, err := s.redis.ZRangeWithScores("wordcloud", 0, -1).Result()
	if err != nil {
		log.Error(err)
		return map[string]float64{}
	}

	wordcloudWords := map[string]float64{}
	for _, value := range words {
		word := fmt.Sprintf("%v", value.Member)

		wordcloudWords[word] = value.Score
		if value.Score == 0 {
			s.redis.ZRem("wordcloud")
		}
	}

	return wordcloudWords
}

func (s *Store) GetTopWords() map[string]float64 {
	words, err := s.redis.ZRevRangeWithScores("wordcloud", 0, 49).Result()
	if err != nil {
		log.Error(err)
		return map[string]float64{}
	}

	wordcloudWords := map[string]float64{}
	for _, value := range words {
		word := fmt.Sprintf("%v", value.Member)
		wordcloudWords[word] = value.Score
	}

	return wordcloudWords
}

func (s *Store) TickDownAll() {
	_, err := s.redis.Eval(`
	local zsetMembers = redis.call('zrange', KEYS[1], '0', '-1')
	for k,member in pairs(zsetMembers) do
		redis.call('zincrby', KEYS[1], -1, member)
	end`,
		[]string{"wordcloud"}).Result()
	if err != nil && err.Error() != "redis: nil" {
		log.Error(err.Error())
	}
}

func (s *Store) RemoveChannel(channelID string) {
	_, err := s.redis.HDel("channels", channelID).Result()
	if err != nil {
		log.Error(err)
	}
	log.Infof("[store] removed %s", channelID)
}

func (s *Store) GetAllChannels() []string {
	channels, err := s.redis.HKeys("channels").Result()
	if err != nil {
		log.Error(err)
		return []string{}
	}

	return channels
}
<commit_msg>reduce way faster<commit_after>package store

import (
	"fmt"

	"github.com/go-redis/redis/v7"
	log "github.com/sirupsen/logrus"
)

type Store struct {
	redis *redis.Client
}

func NewStore() *Store {
	client := redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "",
		DB:       0,
	})

	_, err := client.Ping().Result()
	if err != nil {
		log.Fatal(err)
	}
	log.Info("[store] connection to redis established")

	return &Store{
		redis: client,
	}
}

func (s *Store) UpdateMsgps(channelID string, msgps int64) {
	s.redis.ZAdd("channel:msgps", &redis.Z{Score: float64(msgps), Member: channelID})
}

func (s *Store) GetMsgps(channelID string) float64 {
	score, err := s.redis.ZScore("channel:msgps", channelID).Result()
	if err != nil {
		return 0
	}

	return score
}

func (s *Store) GetMsgpsScores() []redis.Z {
	scores, err := s.redis.ZRevRangeWithScores("channel:msgps", 0, 9).Result()
	if err != nil {
		return []redis.Z{}
	}

	return scores
}

func (s *Store) AddChannels(channelIDs ...string) {
	for _, id := range channelIDs {
		_, err := s.redis.HSet("channels", id, "1").Result()
		if err != nil {
			log.Error(err)
			continue
		}
	}
	log.Infof("[store] added %v", channelIDs)
}

type cloudWord struct {
	Word string
	Add  int64
}

func (s *Store) AddToWordcloud(words ...string) {
	for _, word := range words {
		_, err := s.redis.ZIncrBy("wordcloud", 1, word).Result()
		if err != nil {
			log.Error(err)
			continue
		}
	}
}

func (s *Store) GetEntireWordcloud() map[string]float64 {
	words, err := s.redis.ZRangeWithScores("wordcloud", 0, -1).Result()
	if err != nil {
		log.Error(err)
		return map[string]float64{}
	}

	wordcloudWords := map[string]float64{}
	for _, value := range words {
		word := fmt.Sprintf("%v", value.Member)

		wordcloudWords[word] = value.Score
		if value.Score == 0 {
			s.redis.ZRem("wordcloud")
		}
	}

	return wordcloudWords
}

func (s *Store) GetTopWords() map[string]float64 {
	words, err := s.redis.ZRevRangeWithScores("wordcloud", 0, 49).Result()
	if err != nil {
		log.Error(err)
		return map[string]float64{}
	}

	wordcloudWords := map[string]float64{}
	for _, value := range words {
		word := fmt.Sprintf("%v", value.Member)
		wordcloudWords[word] = value.Score
	}

	return wordcloudWords
}
func (s *Store) TickDownAll() {
	_, err := s.redis.Eval(`
	local zsetMembers = redis.call('zrange', KEYS[1], '0', '-1')
	for k,member in pairs(zsetMembers) do
		redis.call('zincrby', KEYS[1], -1, member)
	end`,
		[]string{"wordcloud"}).Result()
	if err != nil && err.Error() != "redis: nil" {
		log.Error(err.Error())
	}
}

func (s *Store) RemoveChannel(channelID string) {
	_, err := s.redis.HDel("channels", channelID).Result()
	if err != nil {
		log.Error(err)
	}
	log.Infof("[store] removed %s", channelID)
}

func (s *Store) GetAllChannels() []string {
	channels, err := s.redis.HKeys("channels").Result()
	if err != nil {
		log.Error(err)
		return []string{}
	}

	return channels
}
<|endoftext|>


<commit_before>package transfer

import (
	"fmt"
	"hash"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/errors"
	"github.com/github/git-lfs/httputil"
	"github.com/github/git-lfs/localstorage"
	"github.com/github/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

// Adapter for basic HTTP downloads, includes resuming via HTTP Range
type basicDownloadAdapter struct {
	*adapterBase
}

func (a *basicDownloadAdapter) ClearTempStorage() error {
	return os.RemoveAll(a.tempDir())
}

func (a *basicDownloadAdapter) tempDir() string {
	// Must be dedicated to this adapter as deleted by ClearTempStorage
	// Also make local to this repo not global, and separate to localstorage temp,
	// which gets cleared at the end of every invocation
	d := filepath.Join(localstorage.Objects().RootDir, "incomplete")
	if err := os.MkdirAll(d, 0755); err != nil {
		return os.TempDir()
	}
	return d
}

func (a *basicDownloadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
	return nil, nil
}
func (a *basicDownloadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
}

func (a *basicDownloadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {

	f, fromByte, hashSoFar, err := a.checkResumeDownload(t)
	if err != nil {
		return err
	}
	return a.download(t, cb, authOkFunc, f, fromByte, hashSoFar)
}

// Checks to see if a download can be resumed, and if so returns a non-nil locked file, byte start and hash
func (a *basicDownloadAdapter) checkResumeDownload(t *Transfer) (outFile *os.File, fromByte int64, hashSoFar hash.Hash, e error) {
	// lock the file by opening it for read/write, rather than checking Stat() etc
	// which could be subject to race conditions by other processes
	f, err := os.OpenFile(a.downloadFilename(t), os.O_RDWR, 0644)

	if err != nil {
		// Create a new file instead, must not already exist or error (permissions / race condition)
		newfile, err := os.OpenFile(a.downloadFilename(t), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644)
		return newfile, 0, nil, err
	}

	// Successfully opened an existing file at this point
	// Read any existing data into hash then return file handle at end
	hash := tools.NewLfsContentHash()
	n, err := io.Copy(hash, f)
	if err != nil {
		f.Close()
		return nil, 0, nil, err
	}
	tracerx.Printf("xfer: Attempting to resume download of %q from byte %d", t.Object.Oid, n)
	return f, n, hash, nil

}

// Create or open a download file for resuming
func (a *basicDownloadAdapter) downloadFilename(t *Transfer) string {
	// Not a temp file since we will be resuming it
	return filepath.Join(a.tempDir(), t.Object.Oid+".tmp")
}

// download starts or resumes and download. Always closes dlFile if non-nil
func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback, authOkFunc func(), dlFile *os.File, fromByte int64, hash hash.Hash) error {
	if dlFile != nil {
		// ensure we always close dlFile. Note that this does not conflict with the
		// early close below, as close is idempotent.
		defer dlFile.Close()
	}

	rel, ok := t.Object.Rel("download")
	if !ok {
		return errors.New("Object not found on the server.")
	}

	req, err := httputil.NewHttpRequest("GET", rel.Href, rel.Header)
	if err != nil {
		return err
	}

	if fromByte > 0 {
		if dlFile == nil || hash == nil {
			return fmt.Errorf("Cannot restart %v from %d without a file & hash", t.Object.Oid, fromByte)
		}
		// We could just use a start byte, but since we know the length be specific
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", fromByte, t.Object.Size-1))
	}

	res, err := httputil.DoHttpRequest(config.Config, req, t.Object.NeedsAuth())
	if err != nil {
		// Special-case status code 416 () - fall back
		if fromByte > 0 && dlFile != nil && res.StatusCode == 416 {
			tracerx.Printf("xfer: server rejected resume download request for %q from byte %d; re-downloading from start", t.Object.Oid, fromByte)
			dlFile.Close()
			os.Remove(dlFile.Name())
			return a.download(t, cb, authOkFunc, nil, 0, nil)
		}
		return errors.NewRetriableError(err)
	}
	httputil.LogTransfer(config.Config, "lfs.data.download", res)
	defer res.Body.Close()

	// Range request must return 206 & content range to confirm
	if fromByte > 0 {
		rangeRequestOk := false
		var failReason string
		// check 206 and Content-Range, fall back if either not as expected
		if res.StatusCode == 206 {
			// Probably a successful range request, check Content-Range
			if rangeHdr := res.Header.Get("Content-Range"); rangeHdr != "" {
				regex := regexp.MustCompile(`bytes (\d+)\-.*`)
				match := regex.FindStringSubmatch(rangeHdr)
				if match != nil && len(match) > 1 {
					contentStart, _ := strconv.ParseInt(match[1], 10, 64)
					if contentStart == fromByte {
						rangeRequestOk = true
					} else {
						failReason = fmt.Sprintf("Content-Range start byte incorrect: %s expected %d", match[1], fromByte)
					}
				} else {
					failReason = fmt.Sprintf("badly formatted Content-Range header: %q", rangeHdr)
				}
			} else {
				failReason = "missing Content-Range header in response"
			}
		} else {
			failReason = fmt.Sprintf("expected status code 206, received %d", res.StatusCode)
		}
		if rangeRequestOk {
			tracerx.Printf("xfer: server accepted resume download request: %q from byte %d", t.Object.Oid, fromByte)
			advanceCallbackProgress(cb, t, fromByte)
		} else {
			// Abort resume, perform regular download
			tracerx.Printf("xfer: failed to resume download for %q from byte %d: %s. Re-downloading from start", t.Object.Oid, fromByte, failReason)
			dlFile.Close()
			os.Remove(dlFile.Name())
			if res.StatusCode == 200 {
				// If status code was 200 then server just ignored Range header and
				// sent everything. Don't re-request, use this one from byte 0
				dlFile = nil
				fromByte = 0
				hash = nil
			} else {
				// re-request needed
				return a.download(t, cb, authOkFunc, nil, 0, nil)
			}
		}
	}

	// Signal auth OK on success response, before starting download to free up
	// other workers immediately
	if authOkFunc != nil {
		authOkFunc()
	}

	var hasher *tools.HashingReader
	if fromByte > 0 && hash != nil {
		// pre-load hashing reader with previous content
		hasher = tools.NewHashingReaderPreloadHash(res.Body, hash)
	} else {
		hasher = tools.NewHashingReader(res.Body)
	}

	if dlFile == nil {
		// New file start
		dlFile, err = os.OpenFile(a.downloadFilename(t), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if err != nil {
			return err
		}
		defer dlFile.Close()
	}
	dlfilename := dlFile.Name()
	// Wrap callback to give name context
	ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		if cb != nil {
			return cb(t.Name, totalSize, readSoFar+fromByte, readSinceLast)
		}
		return nil
	}
	written, err := tools.CopyWithCallback(dlFile, hasher, res.ContentLength, ccb)
	if err != nil {
		return fmt.Errorf("cannot write data to tempfile %q: %v", dlfilename, err)
	}
	if err := dlFile.Close(); err != nil {
		return fmt.Errorf("can't close tempfile %q: %v", dlfilename, err)
	}

	if actual := hasher.Hash(); actual != t.Object.Oid {
		return fmt.Errorf("Expected OID %s, got %s after %d bytes written", t.Object.Oid, actual, written)
	}

	return tools.RenameFileCopyPermissions(dlfilename, t.Path)
}

func configureBasicDownloadAdapter(m *Manifest) {
	m.RegisterNewTransferAdapterFunc(BasicAdapterName, Download, func(name string, dir Direction) TransferAdapter {
		switch dir {
		case Download:
			bd := &basicDownloadAdapter{newAdapterBase(name, dir, nil)}
			// self implements impl
			bd.transferImpl = bd
			return bd
		case Upload:
			panic("Should never ask this func to upload")
		}
		return nil
	})
}
<commit_msg>retry if file download failed<commit_after>package transfer

import (
	"fmt"
	"hash"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/errors"
	"github.com/github/git-lfs/httputil"
	"github.com/github/git-lfs/localstorage"
	"github.com/github/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

// Adapter for basic HTTP downloads, includes resuming via HTTP Range
type basicDownloadAdapter struct {
	*adapterBase
}

func (a *basicDownloadAdapter) ClearTempStorage() error {
	return os.RemoveAll(a.tempDir())
}

func (a *basicDownloadAdapter) tempDir() string {
	// Must be dedicated to this adapter as deleted by ClearTempStorage
	// Also make local to this repo not global, and separate to localstorage temp,
	// which gets cleared at the end of every invocation
	d := filepath.Join(localstorage.Objects().RootDir, "incomplete")
	if err := os.MkdirAll(d, 0755); err != nil {
		return os.TempDir()
	}
	return d
}

func (a *basicDownloadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
	return nil, nil
}
func (a *basicDownloadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
}

func (a *basicDownloadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {

	f, fromByte, hashSoFar, err := a.checkResumeDownload(t)
	if err != nil {
		return err
	}
	return a.download(t, cb, authOkFunc, f, fromByte, hashSoFar)
}

// Checks to see if a download can be resumed, and if so returns a non-nil locked file, byte start and hash
func (a *basicDownloadAdapter) checkResumeDownload(t *Transfer) (outFile *os.File, fromByte int64, hashSoFar hash.Hash, e error) {
	// lock the file by opening it for read/write, rather than checking Stat() etc
	// which could be subject to race conditions by other processes
	f, err := os.OpenFile(a.downloadFilename(t), os.O_RDWR, 0644)

	if err != nil {
		// Create a new file instead, must not already exist or error (permissions / race condition)
		newfile, err := os.OpenFile(a.downloadFilename(t), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644)
		return newfile, 0, nil, err
	}

	// Successfully opened an existing file at this point
	// Read any existing data into hash then return file handle at end
	hash := tools.NewLfsContentHash()
	n, err := io.Copy(hash, f)
	if err != nil {
		f.Close()
		return nil, 0, nil, err
	}
	tracerx.Printf("xfer: Attempting to resume download of %q from byte %d", t.Object.Oid, n)
	return f, n, hash, nil

}

// Create or open a download file for resuming
func (a *basicDownloadAdapter) downloadFilename(t *Transfer) string {
	// Not a temp file since we will be resuming it
	return filepath.Join(a.tempDir(), t.Object.Oid+".tmp")
}

// download starts or resumes and download. Always closes dlFile if non-nil
func (a *basicDownloadAdapter) download(t *Transfer, cb TransferProgressCallback, authOkFunc func(), dlFile *os.File, fromByte int64, hash hash.Hash) error {
	if dlFile != nil {
		// ensure we always close dlFile. Note that this does not conflict with the
		// early close below, as close is idempotent.
		defer dlFile.Close()
	}

	rel, ok := t.Object.Rel("download")
	if !ok {
		return errors.New("Object not found on the server.")
	}

	req, err := httputil.NewHttpRequest("GET", rel.Href, rel.Header)
	if err != nil {
		return err
	}

	if fromByte > 0 {
		if dlFile == nil || hash == nil {
			return fmt.Errorf("Cannot restart %v from %d without a file & hash", t.Object.Oid, fromByte)
		}
		// We could just use a start byte, but since we know the length be specific
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", fromByte, t.Object.Size-1))
	}

	res, err := httputil.DoHttpRequest(config.Config, req, t.Object.NeedsAuth())
	if err != nil {
		// Special-case status code 416 () - fall back
		if fromByte > 0 && dlFile != nil && res.StatusCode == 416 {
			tracerx.Printf("xfer: server rejected resume download request for %q from byte %d; re-downloading from start", t.Object.Oid, fromByte)
			dlFile.Close()
			os.Remove(dlFile.Name())
			return a.download(t, cb, authOkFunc, nil, 0, nil)
		}
		return errors.NewRetriableError(err)
	}
	httputil.LogTransfer(config.Config, "lfs.data.download", res)
	defer res.Body.Close()

	// Range request must return 206 & content range to confirm
	if fromByte > 0 {
		rangeRequestOk := false
		var failReason string
		// check 206 and Content-Range, fall back if either not as expected
		if res.StatusCode == 206 {
			// Probably a successful range request, check Content-Range
			if rangeHdr := res.Header.Get("Content-Range"); rangeHdr != "" {
				regex := regexp.MustCompile(`bytes (\d+)\-.*`)
				match := regex.FindStringSubmatch(rangeHdr)
				if match != nil && len(match) > 1 {
					contentStart, _ := strconv.ParseInt(match[1], 10, 64)
					if contentStart == fromByte {
						rangeRequestOk = true
					} else {
						failReason = fmt.Sprintf("Content-Range start byte incorrect: %s expected %d", match[1], fromByte)
					}
				} else {
					failReason = fmt.Sprintf("badly formatted Content-Range header: %q", rangeHdr)
				}
			} else {
				failReason = "missing Content-Range header in response"
			}
		} else {
			failReason = fmt.Sprintf("expected status code 206, received %d", res.StatusCode)
		}
		if rangeRequestOk {
			tracerx.Printf("xfer: server accepted resume download request: %q from byte %d", t.Object.Oid, fromByte)
			advanceCallbackProgress(cb, t, fromByte)
		} else {
			// Abort resume, perform regular download
			tracerx.Printf("xfer: failed to resume download for %q from byte %d: %s. Re-downloading from start", t.Object.Oid, fromByte, failReason)
			dlFile.Close()
			os.Remove(dlFile.Name())
			if res.StatusCode == 200 {
				// If status code was 200 then server just ignored Range header and
				// sent everything. Don't re-request, use this one from byte 0
				dlFile = nil
				fromByte = 0
				hash = nil
			} else {
				// re-request needed
				return a.download(t, cb, authOkFunc, nil, 0, nil)
			}
		}
	}

	// Signal auth OK on success response, before starting download to free up
	// other workers immediately
	if authOkFunc != nil {
		authOkFunc()
	}

	var hasher *tools.HashingReader
	httpReader := tools.NewRetriableReader(res.Body)

	if fromByte > 0 && hash != nil {
		// pre-load hashing reader with previous content
		hasher = tools.NewHashingReaderPreloadHash(httpReader, hash)
	} else {
		hasher = tools.NewHashingReader(httpReader)
	}

	if dlFile == nil {
		// New file start
		dlFile, err = os.OpenFile(a.downloadFilename(t), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if err != nil {
			return err
		}
		defer dlFile.Close()
	}
	dlfilename := dlFile.Name()
	// Wrap callback to give name context
	ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		if cb != nil {
			return cb(t.Name, totalSize, readSoFar+fromByte, readSinceLast)
		}
		return nil
	}
	written, err := tools.CopyWithCallback(dlFile, hasher, res.ContentLength, ccb)
	if err != nil {
		wrappedErr := fmt.Errorf("cannot write data to tempfile %q: %v", dlfilename, err)
		if errutil.IsRetriableError(err) {
			return errutil.NewRetriableError(wrappedErr)
		}
		return wrappedErr
	}
	if err := dlFile.Close(); err != nil {
		return fmt.Errorf("can't close tempfile %q: %v", dlfilename, err)
	}

	if actual := hasher.Hash(); actual != t.Object.Oid {
		return fmt.Errorf("Expected OID %s, got %s after %d bytes written", t.Object.Oid, actual, written)
	}

	return tools.RenameFileCopyPermissions(dlfilename, t.Path)
}

func configureBasicDownloadAdapter(m *Manifest) {
	m.RegisterNewTransferAdapterFunc(BasicAdapterName, Download, func(name string, dir Direction) TransferAdapter {
		switch dir {
		case Download:
			bd := &basicDownloadAdapter{newAdapterBase(name, dir, nil)}
			// self implements impl
			bd.transferImpl = bd
			return bd
		case Upload:
			panic("Should never ask this func to upload")
		}
		return nil
	})
}
<|endoftext|>


<commit_before>package tartrans

import (
	"archive/tar"
	"context"
	"crypto/sha512"
	"fmt"
	"io"
	"net/url"
	"os"
	"strings"

	"github.com/polydawn/refmt/misc"

	. "github.com/polydawn/go-errcat"
	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/rio"
	"go.polydawn.net/go-timeless-api/util"
	"go.polydawn.net/rio/config"
	"go.polydawn.net/rio/fs"
	"go.polydawn.net/rio/fs/osfs"
	"go.polydawn.net/rio/fsOp"
	"go.polydawn.net/rio/lib/treewalk"
	"go.polydawn.net/rio/transmat/mixins/cache"
	"go.polydawn.net/rio/transmat/mixins/filters"
	"go.polydawn.net/rio/transmat/mixins/fshash"
	"go.polydawn.net/rio/transmat/util"
	"go.polydawn.net/rio/warehouse/impl/kvfs"
)

var (
	_ rio.UnpackFunc = Unpack
)

func Unpack(
	ctx context.Context, // Long-running call.  Cancellable.
	wareID api.WareID, // What wareID to fetch for unpacking.
	path string, // Where to unpack the fileset (absolute path).
	filt api.FilesetFilters, // Optionally: filters we should apply while unpacking.
	placementMode rio.PlacementMode, // Optionally: a placement mode (default is "copy").
	warehouses []api.WarehouseAddr, // Warehouses we can try to fetch from.
	monitor rio.Monitor, // Optionally: callbacks for progress monitoring.
) (api.WareID, error) {
	// Sanitize arguments.
	if wareID.Type != PackType {
		return api.WareID{}, Errorf(rio.ErrUsage, "this transmat implementation only supports packtype %q (not %q)", PackType, wareID.Type)
	}
	if placementMode == "" {
		placementMode = rio.Placement_Copy
	}
	// Wrap the direct unpack func with cache behavior; call that.
	return cache.Lrn2Cache(
		osfs.New(config.GetCacheBasePath()),
		unpack,
	)(ctx, wareID, path, filt, placementMode, warehouses, monitor)
}

func unpack(
	ctx context.Context,
	wareID api.WareID,
	path string,
	filt api.FilesetFilters,
	placementMode rio.PlacementMode,
	warehouses []api.WarehouseAddr,
	monitor rio.Monitor,
) (api.WareID, error) {
	// Sanitize arguments.
	path2 := fs.MustAbsolutePath(path)
	filt2, err := apiutil.ProcessFilters(filt, apiutil.FilterPurposeUnpack)
	if err != nil {
		return api.WareID{}, Errorf(rio.ErrUsage, "invalid filter specification: %s", err)
	}

	// Pick a warehouse.
	//  With K/V warehouses, this takes the form of "pick the first one that answers".
	var reader io.ReadCloser
	for _, addr := range warehouses {
		// REVIEW ... Do I really have to parse this again?  is this sanely encapsulated?
		u, err := url.Parse(string(addr))
		if err != nil {
			return api.WareID{}, Errorf(rio.ErrUsage, "failed to parse URI: %s", err)
		}
		switch u.Scheme {
		case "file", "ca+file":
			whCtrl, err := kvfs.NewController(addr)
			switch Category(err) {
			case nil:
				// pass
			case rio.ErrWarehouseUnavailable:
				// TODO log something to the monitor
				continue // okay!  skip to the next one.
			default:
				return api.WareID{}, err
			}
			reader, err = whCtrl.OpenReader(wareID)
			switch Category(err) {
			case nil:
				// pass
			case rio.ErrWareNotFound:
				// TODO log something to the monitor
				continue // okay!  skip to the next one.
			default:
				return api.WareID{}, err
			}
		default:
			return api.WareID{}, Errorf(rio.ErrUsage, "tar unpack doesn't support %q scheme (valid options are 'file' or 'ca+file')", u.Scheme)
		}
	}
	if reader == nil { // aka if no warehouses available:
		return api.WareID{}, Errorf(rio.ErrWarehouseUnavailable, "no warehouses were available!")
	}
	defer reader.Close()

	// Wrap input stream with decompression as necessary.
	//  Which kind of decompression to use can be autodetected by magic bytes.
	reader2, err := Decompress(reader)
	if err != nil {
		return api.WareID{}, Errorf(rio.ErrWareCorrupt, "corrupt tar compression: %s", err)
	}

	// Convert the raw byte reader to a tar stream.
	tarReader := tar.NewReader(reader2)

	// Extract.
	gotWare, err := unpackTar(ctx, path2, filt2, tarReader)
	if err != nil {
		return gotWare, err
	}

	// Check for hash mismatch before returning, because that IS an error,
	//  but also return the hash we got either way.
	if gotWare != wareID {
		return gotWare, ErrorDetailed(
			rio.ErrWareHashMismatch,
			fmt.Sprintf("hash mismatch: expected %q, got %q", wareID, gotWare),
			map[string]string{
				"expected": wareID.String(),
				"actual":   gotWare.String(),
			},
		)
	}
	return gotWare, nil
}

func unpackTar(
	ctx context.Context,
	destBasePath fs.AbsolutePath,
	filt apiutil.FilesetFilters,
	tr *tar.Reader,
) (api.WareID, error) {
	// Allocate bucket for keeping each metadata entry and content hash;
	// the full tree hash will be computed from this at the end.
	bucket := &fshash.MemoryBucket{}

	// Construct filesystem wrapper to use for all our ops.
	afs := osfs.New(destBasePath)

	// Iterate over each tar entry, mutating filesystem as we go.
	for {
		fmeta := fs.Metadata{}
		thdr, err := tr.Next()

		// Check for done.
		if err == io.EOF {
			break // sucess!  end of archive.
		}
		if err != nil {
			return api.WareID{}, Errorf(rio.ErrWareCorrupt, "corrupt tar: %s", err)
		}
		if ctx.Err() != nil {
			return api.WareID{}, Errorf(rio.ErrCancelled, "cancelled")
		}

		// Reshuffle metainfo to our default format.
		if err := TarHdrToMetadata(thdr, &fmeta); err != nil {
			return api.WareID{}, err
		}
		if strings.HasPrefix(fmeta.Name.String(), "..") {
			return api.WareID{}, Errorf(rio.ErrWareCorrupt, "corrupt tar: paths that use '../' to leave the base dir are invalid")
		}

		// Apply filters.
		filters.Apply(filt, &fmeta)

		// Infer parents, if necessary.  The tar format allows implicit parent dirs.
		//
		// Note that if any of the implicitly conjured dirs is specified later, unpacking won't notice,
		// but bucket hashing iteration will (correctly) blow up for repeat entries.
		// It may well be possible to construct a tar like that, but it's already well established that
		// tars with repeated filenames are just asking for trouble and shall be rejected without
		// ceremony because they're just a ridiculous idea.
		for parent := fmeta.Name.Dir(); parent != (fs.RelPath{}); parent = parent.Dir() {
			_, err := os.Lstat(destBasePath.Join(parent).String())
			// if it already exists, move along; if the error is anything interesting, let PlaceFile decide how to deal with it
			if err == nil || !os.IsNotExist(err) {
				continue
			}
			// if we're missing a dir, conjure a node with defaulted values (same as we do for "./")
			conjuredFmeta := fshash.DefaultDirMetadata()
			conjuredFmeta.Name = parent
			if err := fsOp.PlaceFile(afs, conjuredFmeta, nil, false); err != nil {
				return api.WareID{}, err // FIXME these errors should be category'd here
			}
			bucket.AddRecord(conjuredFmeta, nil)
		}

		// Place the file.
		switch fmeta.Type {
		case fs.Type_File:
			reader := &util.HashingReader{tr, sha512.New384()}
			if err := fsOp.PlaceFile(afs, fmeta, reader, false); err != nil {
				return api.WareID{}, err // FIXME these errors should be category'd here
			}
			bucket.AddRecord(fmeta, reader.Hasher.Sum(nil))
		default:
			if err := fsOp.PlaceFile(afs, fmeta, nil, false); err != nil {
				return api.WareID{}, err // FIXME these errors should be category'd here
			}
			bucket.AddRecord(fmeta, nil)
		}
	}

	// Cleanup dir times with a post-order traversal over the bucket.
	//  Files and dirs placed inside dirs cause the parent's mtime to update, so we have to re-pave them.
	if err := treewalk.Walk(bucket.Iterator(), nil, func(node treewalk.Node) error {
		record := node.(fshash.RecordIterator).Record()
		if record.Metadata.Type != fs.Type_Dir {
			return nil
		}
		return afs.SetTimesNano(record.Metadata.Name, record.Metadata.Mtime, fs.DefaultAtime)
	}); err != nil {
		return api.WareID{}, err // FIXME these errors should be category'd here
	}
	// Bucket processing may have created a root node if missing.  If so, make sure we apply its props (all of them, not just time).
	if err := fsOp.PlaceFile(afs, bucket.Root().Metadata, nil, false); err != nil {
		return api.WareID{}, err // FIXME these errors should be category'd here
	}

	// Hash the thing!
	hash := fshash.HashBucket(bucket, sha512.New384)

	return api.WareID{"tar", misc.Base58Encode(hash)}, nil
}
<commit_msg>transmat/tar: wire in kvhttp warehouse support.<commit_after>package tartrans

import (
	"archive/tar"
	"context"
	"crypto/sha512"
	"fmt"
	"io"
	"net/url"
	"os"
	"strings"

	"github.com/polydawn/refmt/misc"

	. "github.com/polydawn/go-errcat"
	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/rio"
	"go.polydawn.net/go-timeless-api/util"
	"go.polydawn.net/rio/config"
	"go.polydawn.net/rio/fs"
	"go.polydawn.net/rio/fs/osfs"
	"go.polydawn.net/rio/fsOp"
	"go.polydawn.net/rio/lib/treewalk"
	"go.polydawn.net/rio/transmat/mixins/cache"
	"go.polydawn.net/rio/transmat/mixins/filters"
	"go.polydawn.net/rio/transmat/mixins/fshash"
	"go.polydawn.net/rio/transmat/util"
	"go.polydawn.net/rio/warehouse"
	"go.polydawn.net/rio/warehouse/impl/kvfs"
	"go.polydawn.net/rio/warehouse/impl/kvhttp"
)

var (
	_ rio.UnpackFunc = Unpack
)

func Unpack(
	ctx context.Context, // Long-running call.  Cancellable.
	wareID api.WareID, // What wareID to fetch for unpacking.
	path string, // Where to unpack the fileset (absolute path).
	filt api.FilesetFilters, // Optionally: filters we should apply while unpacking.
	placementMode rio.PlacementMode, // Optionally: a placement mode (default is "copy").
	warehouses []api.WarehouseAddr, // Warehouses we can try to fetch from.
	monitor rio.Monitor, // Optionally: callbacks for progress monitoring.
) (api.WareID, error) {
	// Sanitize arguments.
	if wareID.Type != PackType {
		return api.WareID{}, Errorf(rio.ErrUsage, "this transmat implementation only supports packtype %q (not %q)", PackType, wareID.Type)
	}
	if placementMode == "" {
		placementMode = rio.Placement_Copy
	}
	// Wrap the direct unpack func with cache behavior; call that.
	return cache.Lrn2Cache(
		osfs.New(config.GetCacheBasePath()),
		unpack,
	)(ctx, wareID, path, filt, placementMode, warehouses, monitor)
}

func unpack(
	ctx context.Context,
	wareID api.WareID,
	path string,
	filt api.FilesetFilters,
	placementMode rio.PlacementMode,
	warehouses []api.WarehouseAddr,
	monitor rio.Monitor,
) (api.WareID, error) {
	// Sanitize arguments.
	path2 := fs.MustAbsolutePath(path)
	filt2, err := apiutil.ProcessFilters(filt, apiutil.FilterPurposeUnpack)
	if err != nil {
		return api.WareID{}, Errorf(rio.ErrUsage, "invalid filter specification: %s", err)
	}

	// Pick a warehouse.
	//  With K/V warehouses, this takes the form of "pick the first one that answers".
	var reader io.ReadCloser
	for _, addr := range warehouses {
		// REVIEW ... Do I really have to parse this again?  is this sanely encapsulated?
		u, err := url.Parse(string(addr))
		if err != nil {
			return api.WareID{}, Errorf(rio.ErrUsage, "failed to parse URI: %s", err)
		}
		var whCtrl warehouse.BlobstoreController
		switch u.Scheme {
		case "file", "ca+file":
			whCtrl, err = kvfs.NewController(addr)
		case "http", "ca+http", "https", "ca+https":
			whCtrl, err = kvhttp.NewController(addr)
		default:
			return api.WareID{}, Errorf(rio.ErrUsage, "tar unpack doesn't support %q scheme (valid options are 'file', 'ca+file', 'http', 'ca+http', 'https', or 'ca+https')", u.Scheme)
		}
		switch Category(err) {
		case nil:
			// pass
		case rio.ErrWarehouseUnavailable:
			// TODO log something to the monitor
			continue // okay!  skip to the next one.
		default:
			return api.WareID{}, err
		}
		reader, err = whCtrl.OpenReader(wareID)
		switch Category(err) {
		case nil:
			// pass
		case rio.ErrWareNotFound:
			// TODO log something to the monitor
			continue // okay!  skip to the next one.
		default:
			return api.WareID{}, err
		}
	}
	if reader == nil { // aka if no warehouses available:
		return api.WareID{}, Errorf(rio.ErrWarehouseUnavailable, "no warehouses were available!")
	}
	defer reader.Close()

	// Wrap input stream with decompression as necessary.
	//  Which kind of decompression to use can be autodetected by magic bytes.
	reader2, err := Decompress(reader)
	if err != nil {
		return api.WareID{}, Errorf(rio.ErrWareCorrupt, "corrupt tar compression: %s", err)
	}

	// Convert the raw byte reader to a tar stream.
	tarReader := tar.NewReader(reader2)

	// Extract.
	gotWare, err := unpackTar(ctx, path2, filt2, tarReader)
	if err != nil {
		return gotWare, err
	}

	// Check for hash mismatch before returning, because that IS an error,
	//  but also return the hash we got either way.
	if gotWare != wareID {
		return gotWare, ErrorDetailed(
			rio.ErrWareHashMismatch,
			fmt.Sprintf("hash mismatch: expected %q, got %q", wareID, gotWare),
			map[string]string{
				"expected": wareID.String(),
				"actual":   gotWare.String(),
			},
		)
	}
	return gotWare, nil
}

func unpackTar(
	ctx context.Context,
	destBasePath fs.AbsolutePath,
	filt apiutil.FilesetFilters,
	tr *tar.Reader,
) (api.WareID, error) {
	// Allocate bucket for keeping each metadata entry and content hash;
	// the full tree hash will be computed from this at the end.
	bucket := &fshash.MemoryBucket{}

	// Construct filesystem wrapper to use for all our ops.
	afs := osfs.New(destBasePath)

	// Iterate over each tar entry, mutating filesystem as we go.
	for {
		fmeta := fs.Metadata{}
		thdr, err := tr.Next()

		// Check for done.
		if err == io.EOF {
			break // sucess!  end of archive.
		}
		if err != nil {
			return api.WareID{}, Errorf(rio.ErrWareCorrupt, "corrupt tar: %s", err)
		}
		if ctx.Err() != nil {
			return api.WareID{}, Errorf(rio.ErrCancelled, "cancelled")
		}

		// Reshuffle metainfo to our default format.
		if err := TarHdrToMetadata(thdr, &fmeta); err != nil {
			return api.WareID{}, err
		}
		if strings.HasPrefix(fmeta.Name.String(), "..") {
			return api.WareID{}, Errorf(rio.ErrWareCorrupt, "corrupt tar: paths that use '../' to leave the base dir are invalid")
		}

		// Apply filters.
		filters.Apply(filt, &fmeta)

		// Infer parents, if necessary.  The tar format allows implicit parent dirs.
		//
		// Note that if any of the implicitly conjured dirs is specified later, unpacking won't notice,
		// but bucket hashing iteration will (correctly) blow up for repeat entries.
		// It may well be possible to construct a tar like that, but it's already well established that
		// tars with repeated filenames are just asking for trouble and shall be rejected without
		// ceremony because they're just a ridiculous idea.
		for parent := fmeta.Name.Dir(); parent != (fs.RelPath{}); parent = parent.Dir() {
			_, err := os.Lstat(destBasePath.Join(parent).String())
			// if it already exists, move along; if the error is anything interesting, let PlaceFile decide how to deal with it
			if err == nil || !os.IsNotExist(err) {
				continue
			}
			// if we're missing a dir, conjure a node with defaulted values (same as we do for "./")
			conjuredFmeta := fshash.DefaultDirMetadata()
			conjuredFmeta.Name = parent
			if err := fsOp.PlaceFile(afs, conjuredFmeta, nil, false); err != nil {
				return api.WareID{}, err // FIXME these errors should be category'd here
			}
			bucket.AddRecord(conjuredFmeta, nil)
		}

		// Place the file.
		switch fmeta.Type {
		case fs.Type_File:
			reader := &util.HashingReader{tr, sha512.New384()}
			if err := fsOp.PlaceFile(afs, fmeta, reader, false); err != nil {
				return api.WareID{}, err // FIXME these errors should be category'd here
			}
			bucket.AddRecord(fmeta, reader.Hasher.Sum(nil))
		default:
			if err := fsOp.PlaceFile(afs, fmeta, nil, false); err != nil {
				return api.WareID{}, err // FIXME these errors should be category'd here
			}
			bucket.AddRecord(fmeta, nil)
		}
	}

	// Cleanup dir times with a post-order traversal over the bucket.
	//  Files and dirs placed inside dirs cause the parent's mtime to update, so we have to re-pave them.
	if err := treewalk.Walk(bucket.Iterator(), nil, func(node treewalk.Node) error {
		record := node.(fshash.RecordIterator).Record()
		if record.Metadata.Type != fs.Type_Dir {
			return nil
		}
		return afs.SetTimesNano(record.Metadata.Name, record.Metadata.Mtime, fs.DefaultAtime)
	}); err != nil {
		return api.WareID{}, err // FIXME these errors should be category'd here
	}
	// Bucket processing may have created a root node if missing.  If so, make sure we apply its props (all of them, not just time).
	if err := fsOp.PlaceFile(afs, bucket.Root().Metadata, nil, false); err != nil {
		return api.WareID{}, err // FIXME these errors should be category'd here
	}

	// Hash the thing!
	hash := fshash.HashBucket(bucket, sha512.New384)

	return api.WareID{"tar", misc.Base58Encode(hash)}, nil
}
<|endoftext|>


<commit_before>package main
//编写一个函数，原地将一个UTF-8编码的[]byte类型的slice中相邻的空格（参考unicode.IsSpace）替换成一个空格返回
import (
	"fmt"
	"unicode/utf8"
	"unicode"
)

func main() {
	s := []byte("我有    很   多  的   空  格      ")
	s1 :=[]byte("我没有空格")
	s2 := []byte("    ")
	s3 := []byte("    我只有前面有空格")
	s4 := []byte("我只有后面有空格    ")

	clearSpace(s)
	clearSpace(s1)
	clearSpace(s2)
	clearSpace(s3)
	clearSpace(s4)
}

func clearSpace(b []byte) {
	count := 0
	for i := 0; i < len(b); {
		r, size := utf8.DecodeRune(b[i:])
		if (unicode.IsSpace(r)) {
			start_ := i + size
			end_ := start_
			for j := start_; j < start_ + len(b[start_:]); {
				c, cSize := utf8.DecodeRune(b[j:])
				if unicode.IsSpace(c) {
					end_ += cSize
				} else {
					break
				}
				j += cSize
			}
			len_ := end_- start_
			if len_ > count {
				count = len_
			}
			for j := start_; j < len(b); j++ {
				if (j+len_ < len(b)) {
					b[j] = b[j+len_]
				}
			}
		}
		i += size
	}
	fmt.Println(string(b[:len(b) - count]),count, len(string(b[:len(b) - count])))
}
<commit_msg>优化copy代码<commit_after>package main
//编写一个函数，原地将一个UTF-8编码的[]byte类型的slice中相邻的空格（参考unicode.IsSpace）替换成一个空格返回
import (
	"fmt"
	"unicode/utf8"
	"unicode"
)

func main() {
	s := []byte("我有    很   多  的   空  格      ")
	s1 :=[]byte("我没有空格")
	s2 := []byte("    ")
	s3 := []byte("    我只有前面有空格")
	s4 := []byte("我只有后面有空格    ")

	clearSpace(s)
	clearSpace(s1)
	clearSpace(s2)
	clearSpace(s3)
	clearSpace(s4)
}

func clearSpace(b []byte) {
	count := 0
	for i := 0; i < len(b); {
		r, size := utf8.DecodeRune(b[i:])
		if (unicode.IsSpace(r)) {
			start_ := i + size
			end_ := start_
			for j := start_; j < start_ + len(b[start_:]); {
				c, cSize := utf8.DecodeRune(b[j:])
				if unicode.IsSpace(c) {
					end_ += cSize
				} else {
					break
				}
				j += cSize
			}
			len_ := end_- start_
			// 多余的空格数
			if len_ > count {
				count = len_
			}
			// 优化copy代码
			copy(b[start_:], b[end_:])
			// for j := start_; j < len(b); j++ {
			//	if (j+len_ < len(b)) {
			//		b[j] = b[j+len_]
			//	}
			// }
		}
		i += size
	}
	fmt.Println(string(b[:len(b) - count]),count, len(string(b[:len(b) - count])))
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"
	"io"

	"github.com/googlecloudplatform/gcsfuse/gcsproxy"
	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

type FileInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket
	leaser lease.FileLeaser
	clock  timeutil.Clock

	/////////////////////////
	// Constant data
	/////////////////////////

	id           fuseops.InodeID
	name         string
	attrs        fuseops.InodeAttributes
	gcsChunkSize uint64

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A mutex that must be held when calling certain methods. See documentation
	// for each method.
	mu syncutil.InvariantMutex

	// GUARDED_BY(mu)
	lc lookupCount

	// The source object from which this inode derives.
	//
	// INVARIANT: src.Name == name
	//
	// GUARDED_BY(mu)
	src gcs.Object

	// The current content of this inode, branched from the source object.
	//
	// INVARIANT: content.CheckInvariants() does not panic
	//
	// GUARDED_BY(mu)
	content gcsproxy.MutableContent

	// Has Destroy been called?
	//
	// GUARDED_BY(mu)
	destroyed bool
}

var _ Inode = &FileInode{}

// Create a file inode for the given object in GCS. The initial lookup count is
// zero.
//
// gcsChunkSize controls the maximum size of each individual read request made
// to GCS.
//
// REQUIRES: o != nil
// REQUIRES: o.Generation > 0
// REQUIRES: len(o.Name) > 0
// REQUIRES: o.Name[len(o.Name)-1] != '/'
func NewFileInode(
	id fuseops.InodeID,
	o *gcs.Object,
	attrs fuseops.InodeAttributes,
	gcsChunkSize uint64,
	bucket gcs.Bucket,
	leaser lease.FileLeaser,
	clock timeutil.Clock) (f *FileInode) {
	// Set up the basic struct.
	f = &FileInode{
		bucket:       bucket,
		leaser:       leaser,
		clock:        clock,
		id:           id,
		name:         o.Name,
		attrs:        attrs,
		gcsChunkSize: gcsChunkSize,
		src:          *o,
		content: gcsproxy.NewMutableContent(
			gcsproxy.NewReadProxy(
				o,
				nil, // Initial read lease
				gcsChunkSize,
				leaser,
				bucket),
			clock),
	}

	f.lc.Init(id)

	// Set up invariant checking.
	f.mu = syncutil.NewInvariantMutex(f.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) checkInvariants() {
	if f.destroyed {
		return
	}

	// Make sure the name is legal.
	name := f.Name()
	if len(name) == 0 || name[len(name)-1] == '/' {
		panic("Illegal file name: " + name)
	}

	// INVARIANT: src.Name == name
	if f.src.Name != name {
		panic(fmt.Sprintf("Name mismatch: %q vs. %q", f.src.Name, name))
	}

	// INVARIANT: content.CheckInvariants() does not panic
	f.content.CheckInvariants()
}

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) clobbered(ctx context.Context) (b bool, err error) {
	// Stat the object in GCS.
	req := &gcs.StatObjectRequest{Name: f.name}
	o, err := f.bucket.StatObject(ctx, req)

	// Special case: "not found" means we have been clobbered.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
		b = true
		return
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	// We are clobbered iff the generation doesn't match our source generation.
	b = (o.Generation != f.src.Generation)

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (f *FileInode) Lock() {
	f.mu.Lock()
}

func (f *FileInode) Unlock() {
	f.mu.Unlock()
}

func (f *FileInode) ID() fuseops.InodeID {
	return f.id
}

func (f *FileInode) Name() string {
	return f.name
}

// Return the object generation number from which this inode was branched.
//
// LOCKS_REQUIRED(f)
func (f *FileInode) SourceGeneration() int64 {
	return f.src.Generation
}

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) IncrementLookupCount() {
	f.lc.Inc()
}

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) DecrementLookupCount(n uint64) (destroy bool) {
	destroy = f.lc.Dec(n)
	return
}

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Destroy() (err error) {
	f.destroyed = true

	f.content.Destroy()
	return
}

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Attributes(
	ctx context.Context) (attrs fuseops.InodeAttributes, err error) {
	// Stat the content.
	sr, err := f.content.Stat(ctx)
	if err != nil {
		err = fmt.Errorf("Stat: %v", err)
		return
	}

	// Fill out the struct.
	attrs = f.attrs
	attrs.Size = uint64(sr.Size)

	if sr.Mtime != nil {
		attrs.Mtime = *sr.Mtime
	} else {
		attrs.Mtime = f.src.Updated
	}

	// If the object has been clobbered, we reflect that as the inode being
	// unlinked.
	clobbered, err := f.clobbered(ctx)
	if err != nil {
		err = fmt.Errorf("clobbered: %v", err)
		return
	}

	if !clobbered {
		attrs.Nlink = 1
	}

	return
}

// Serve a read for this file with semantics matching fuseops.ReadFileOp.
//
// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Read(
	ctx context.Context,
	offset int64,
	size int) (data []byte, err error) {
	// Read from the mutable content.
	data = make([]byte, size)
	n, err := f.content.ReadAt(ctx, data, offset)
	data = data[:n]

	// We don't return errors for EOF. Otherwise, propagate errors.
	if err == io.EOF {
		err = nil
	} else if err != nil {
		err = fmt.Errorf("ReadAt: %v", err)
		return
	}

	return
}

// Serve a write for this file with semantics matching fuseops.WriteFileOp.
//
// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Write(
	ctx context.Context,
	data []byte,
	offset int64) (err error) {
	// Write to the mutable content. Note that the mutable content guarantees
	// that it returns an error for short writes.
	_, err = f.content.WriteAt(ctx, data, offset)

	return
}

// Write out contents to GCS. If this fails due to the generation having been
// clobbered, treat it as a non-error (simulating the inode having been
// unlinked).
//
// After this method succeeds, SourceGeneration will return the new generation
// by which this inode should be known (which may be the same as before). If it
// fails, the generation will not change.
//
// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Sync(ctx context.Context) (err error) {
	// Write out the contents if they are dirty.
	rl, newObj, err := gcsproxy.Sync(
		ctx,
		&f.src,
		f.content,
		f.bucket)

	// Special case: a precondition error means we were clobbered, which we treat
	// as being unlinked. There's no reason to return an error in that case.
	if _, ok := err.(*gcs.PreconditionError); ok {
		err = nil
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("gcsproxy.Sync: %v", err)
		return
	}

	// If we wrote out a new object, we need to update our state.
	if newObj != nil {
		f.src = *newObj
		f.content = gcsproxy.NewMutableContent(
			gcsproxy.NewReadProxy(
				newObj,
				rl,
				f.gcsChunkSize,
				f.leaser,
				f.bucket),
			f.clock)
	}

	return
}

// Truncate the file to the specified size.
//
// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Truncate(
	ctx context.Context,
	size int64) (err error) {
	err = f.content.Truncate(ctx, size)
	return
}
<commit_msg>Fixed package inode.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"
	"io"

	"github.com/googlecloudplatform/gcsfuse/gcsproxy"
	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/mutable"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

type FileInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket
	leaser lease.FileLeaser
	clock  timeutil.Clock

	/////////////////////////
	// Constant data
	/////////////////////////

	id           fuseops.InodeID
	name         string
	attrs        fuseops.InodeAttributes
	gcsChunkSize uint64

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A mutex that must be held when calling certain methods. See documentation
	// for each method.
	mu syncutil.InvariantMutex

	// GUARDED_BY(mu)
	lc lookupCount

	// The source object from which this inode derives.
	//
	// INVARIANT: src.Name == name
	//
	// GUARDED_BY(mu)
	src gcs.Object

	// The current content of this inode, branched from the source object.
	//
	// INVARIANT: content.CheckInvariants() does not panic
	//
	// GUARDED_BY(mu)
	content mutable.Content

	// Has Destroy been called?
	//
	// GUARDED_BY(mu)
	destroyed bool
}

var _ Inode = &FileInode{}

// Create a file inode for the given object in GCS. The initial lookup count is
// zero.
//
// gcsChunkSize controls the maximum size of each individual read request made
// to GCS.
//
// REQUIRES: o != nil
// REQUIRES: o.Generation > 0
// REQUIRES: len(o.Name) > 0
// REQUIRES: o.Name[len(o.Name)-1] != '/'
func NewFileInode(
	id fuseops.InodeID,
	o *gcs.Object,
	attrs fuseops.InodeAttributes,
	gcsChunkSize uint64,
	bucket gcs.Bucket,
	leaser lease.FileLeaser,
	clock timeutil.Clock) (f *FileInode) {
	// Set up the basic struct.
	f = &FileInode{
		bucket:       bucket,
		leaser:       leaser,
		clock:        clock,
		id:           id,
		name:         o.Name,
		attrs:        attrs,
		gcsChunkSize: gcsChunkSize,
		src:          *o,
		content: mutable.NewContent(
			gcsproxy.NewReadProxy(
				o,
				nil, // Initial read lease
				gcsChunkSize,
				leaser,
				bucket),
			clock),
	}

	f.lc.Init(id)

	// Set up invariant checking.
	f.mu = syncutil.NewInvariantMutex(f.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) checkInvariants() {
	if f.destroyed {
		return
	}

	// Make sure the name is legal.
	name := f.Name()
	if len(name) == 0 || name[len(name)-1] == '/' {
		panic("Illegal file name: " + name)
	}

	// INVARIANT: src.Name == name
	if f.src.Name != name {
		panic(fmt.Sprintf("Name mismatch: %q vs. %q", f.src.Name, name))
	}

	// INVARIANT: content.CheckInvariants() does not panic
	f.content.CheckInvariants()
}

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) clobbered(ctx context.Context) (b bool, err error) {
	// Stat the object in GCS.
	req := &gcs.StatObjectRequest{Name: f.name}
	o, err := f.bucket.StatObject(ctx, req)

	// Special case: "not found" means we have been clobbered.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
		b = true
		return
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	// We are clobbered iff the generation doesn't match our source generation.
	b = (o.Generation != f.src.Generation)

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (f *FileInode) Lock() {
	f.mu.Lock()
}

func (f *FileInode) Unlock() {
	f.mu.Unlock()
}

func (f *FileInode) ID() fuseops.InodeID {
	return f.id
}

func (f *FileInode) Name() string {
	return f.name
}

// Return the object generation number from which this inode was branched.
//
// LOCKS_REQUIRED(f)
func (f *FileInode) SourceGeneration() int64 {
	return f.src.Generation
}

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) IncrementLookupCount() {
	f.lc.Inc()
}

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) DecrementLookupCount(n uint64) (destroy bool) {
	destroy = f.lc.Dec(n)
	return
}

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Destroy() (err error) {
	f.destroyed = true

	f.content.Destroy()
	return
}

// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Attributes(
	ctx context.Context) (attrs fuseops.InodeAttributes, err error) {
	// Stat the content.
	sr, err := f.content.Stat(ctx)
	if err != nil {
		err = fmt.Errorf("Stat: %v", err)
		return
	}

	// Fill out the struct.
	attrs = f.attrs
	attrs.Size = uint64(sr.Size)

	if sr.Mtime != nil {
		attrs.Mtime = *sr.Mtime
	} else {
		attrs.Mtime = f.src.Updated
	}

	// If the object has been clobbered, we reflect that as the inode being
	// unlinked.
	clobbered, err := f.clobbered(ctx)
	if err != nil {
		err = fmt.Errorf("clobbered: %v", err)
		return
	}

	if !clobbered {
		attrs.Nlink = 1
	}

	return
}

// Serve a read for this file with semantics matching fuseops.ReadFileOp.
//
// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Read(
	ctx context.Context,
	offset int64,
	size int) (data []byte, err error) {
	// Read from the mutable content.
	data = make([]byte, size)
	n, err := f.content.ReadAt(ctx, data, offset)
	data = data[:n]

	// We don't return errors for EOF. Otherwise, propagate errors.
	if err == io.EOF {
		err = nil
	} else if err != nil {
		err = fmt.Errorf("ReadAt: %v", err)
		return
	}

	return
}

// Serve a write for this file with semantics matching fuseops.WriteFileOp.
//
// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Write(
	ctx context.Context,
	data []byte,
	offset int64) (err error) {
	// Write to the mutable content. Note that the mutable content guarantees
	// that it returns an error for short writes.
	_, err = f.content.WriteAt(ctx, data, offset)

	return
}

// Write out contents to GCS. If this fails due to the generation having been
// clobbered, treat it as a non-error (simulating the inode having been
// unlinked).
//
// After this method succeeds, SourceGeneration will return the new generation
// by which this inode should be known (which may be the same as before). If it
// fails, the generation will not change.
//
// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Sync(ctx context.Context) (err error) {
	// Write out the contents if they are dirty.
	rl, newObj, err := gcsproxy.Sync(
		ctx,
		&f.src,
		f.content,
		f.bucket)

	// Special case: a precondition error means we were clobbered, which we treat
	// as being unlinked. There's no reason to return an error in that case.
	if _, ok := err.(*gcs.PreconditionError); ok {
		err = nil
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("gcsproxy.Sync: %v", err)
		return
	}

	// If we wrote out a new object, we need to update our state.
	if newObj != nil {
		f.src = *newObj
		f.content = mutable.NewContent(
			gcsproxy.NewReadProxy(
				newObj,
				rl,
				f.gcsChunkSize,
				f.leaser,
				f.bucket),
			f.clock)
	}

	return
}

// Truncate the file to the specified size.
//
// LOCKS_REQUIRED(f.mu)
func (f *FileInode) Truncate(
	ctx context.Context,
	size int64) (err error) {
	err = f.content.Truncate(ctx, size)
	return
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package main

import (
	"flag"
	"fmt"
	"mig"
	"mig/client"
	"os"
	"os/signal"
	"time"
)

func usage() {
	fmt.Printf(`%s - Mozilla InvestiGator command line client
usage: %s <module> <global options> <module parameters>

--- Global options ---

-c <path>	path to an alternative config file. If not set, use ~/.migrc
-e <duration>	time after which the action expires. 60 seconds by default.
		example: -e 300s (5 minutes)
-i <file>	load and run action from a file. supersedes other action flags.
-show <mode>	type of results to show. if not set, default is 'found'.
		* found: 	only print positive results
		* notfound: 	only print negative results
		* all: 		print all results
-t <target>	target to launch the action on. Defaults to all active agents.
		examples:
		* linux agents:          -t "queueloc LIKE 'linux.%%'"
		* agents named *mysql*:  -t "name like '%%mysql%%'"
		* proxied linux agents:  -t "queueloc LIKE 'linux.%%' AND environment->>'isproxied' = 'true'"
		* agents operated by IT: -t "tags#>>'{operator}'='IT'"

--- Modules documentation ---
Each module provides its own set of parameters. Module parameters must be set *after*
global options for the parsing to work correctly. The following modules are available:
`, os.Args[0], os.Args[0])
	for module, _ := range mig.AvailableModules {
		fmt.Printf("* %s\n", module)
	}
	fmt.Printf("To access a module documentation, use: %s <module> help\n", os.Args[0])
	os.Exit(1)
}

func continueOnFlagError() {
	return
}

func main() {
	var (
		conf                                   client.Configuration
		cli                                    client.Client
		err                                    error
		op                                     mig.Operation
		a                                      mig.Action
		migrc, show, target, expiration, afile string
		modargs                                []string
		modRunner                              interface{}
	)
	defer func() {
		if e := recover(); e != nil {
			fmt.Fprintf(os.Stderr, "FATAL: %v\n", e)
		}
	}()
	homedir := client.FindHomedir()
	fs := flag.NewFlagSet("mig flag", flag.ContinueOnError)
	fs.Usage = continueOnFlagError
	fs.StringVar(&migrc, "c", homedir+"/.migrc", "alternative configuration file")
	fs.StringVar(&show, "show", "found", "type of results to show")
	fs.StringVar(&target, "t", `status='online'`, "action target")
	fs.StringVar(&expiration, "e", "300s", "expiration")
	fs.StringVar(&afile, "i", "/path/to/file", "Load action from file")

	// if first argument is missing, or is help, print help
	// otherwise, pass the remainder of the arguments to the module for parsing
	// this client is agnostic to module parameters
	if len(os.Args) < 2 || os.Args[1] == "help" || os.Args[1] == "-h" || os.Args[1] == "--help" {
		usage()
	}

	// when reading the action from a file, go directly to launch
	if os.Args[1] == "-i" {
		err = fs.Parse(os.Args[1:])
		if err != nil {
			panic(err)
		}
		if afile == "/path/to/file" {
			panic("-i flag must take an action file path as argument")
		}
		a, err = mig.ActionFromFile(afile)
		if err != nil {
			panic(err)
		}
		goto readytolaunch
	}

	// arguments parsing works as follow:
	// * os.Args[1] must contain the name of the module to launch. we first verify
	//   that a module exist for this name and then continue parsing
	// * os.Args[2:] contains both global options and module parameters. We parse the
	//   whole []string to extract global options, and module parameters will be left
	//   unparsed in fs.Args()
	// * fs.Args() with the module parameters is passed as a string to the module parser
	//   which will return a module operation to store in the action
	op.Module = os.Args[1]
	if _, ok := mig.AvailableModules[op.Module]; !ok {
		panic("Unknown module " + op.Module)
	}

	err = fs.Parse(os.Args[2:])
	if err != nil {
		// ignore the flag not defined error, which is expected because
		// module parameters are defined in modules and not in main
		if len(err.Error()) > 30 && err.Error()[0:29] == "flag provided but not defined" {
			// requeue the parameter that failed
			modargs = append(modargs, err.Error()[31:])
		} else {
			// if it's another error, panic
			panic(err)
		}
	}
	for _, arg := range fs.Args() {
		modargs = append(modargs, arg)
	}
	modRunner = mig.AvailableModules[op.Module]()
	if _, ok := modRunner.(mig.HasParamsParser); !ok {
		fmt.Fprintf(os.Stderr, "[error] module '%s' does not support command line invocation\n", op.Module)
		os.Exit(2)
	}
	op.Parameters, err = modRunner.(mig.HasParamsParser).ParamsParser(modargs)
	if err != nil || op.Parameters == nil {
		panic(err)
	}
	a.Operations = append(a.Operations, op)

	a.Name = op.Module + " on '" + target + "'"
	a.Target = target

readytolaunch:
	// instanciate an API client
	conf, err = client.ReadConfiguration(migrc)
	if err != nil {
		panic(err)
	}
	cli = client.NewClient(conf)

	// set the validity 60 second in the past to deal with clock skew
	a.ValidFrom = time.Now().Add(-60 * time.Second).UTC()
	period, err := time.ParseDuration(expiration)
	if err != nil {
		panic(err)
	}
	a.ExpireAfter = a.ValidFrom.Add(period)
	// add extra 60 seconds taken for clock skew
	a.ExpireAfter = a.ExpireAfter.Add(60 * time.Second).UTC()

	asig, err := cli.SignAction(a)
	if err != nil {
		panic(err)
	}
	a = asig

	// evaluate target before launch, give a change to cancel before going out to agents
	agents, err := cli.EvaluateAgentTarget(a.Target)
	if err != nil {
		panic(err)
	}
	fmt.Fprintf(os.Stderr, "%d agents will be targeted. ctrl+c to cancel. launching in ", len(agents))
	for i := 5; i > 0; i-- {
		time.Sleep(1 * time.Second)
		fmt.Fprintf(os.Stderr, "%d ", i)
	}
	fmt.Fprintf(os.Stderr, "GO\n")

	// launch and follow
	a, err = cli.PostAction(a)
	if err != nil {
		panic(err)
	}
	c := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		err = cli.FollowAction(a)
		if err != nil {
			panic(err)
		}
		done <- true
	}()
	select {
	case <-c:
		fmt.Fprintf(os.Stderr, "stop following action. agents may still be running. printing available results:\n")
		goto printresults
	case <-done:
		goto printresults
	}
printresults:
	err = cli.PrintActionResults(a, show)
	if err != nil {
		panic(err)
	}
}
<commit_msg>[minor] ignore incorrect flag error (from upstream package) in command line<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package main

import (
	"flag"
	"fmt"
	"mig"
	"mig/client"
	"os"
	"os/signal"
	"time"
)

func usage() {
	fmt.Printf(`%s - Mozilla InvestiGator command line client
usage: %s <module> <global options> <module parameters>

--- Global options ---

-c <path>	path to an alternative config file. If not set, use ~/.migrc
-e <duration>	time after which the action expires. 60 seconds by default.
		example: -e 300s (5 minutes)
-i <file>	load and run action from a file. supersedes other action flags.
-show <mode>	type of results to show. if not set, default is 'found'.
		* found: 	only print positive results
		* notfound: 	only print negative results
		* all: 		print all results
-t <target>	target to launch the action on. Defaults to all active agents.
		examples:
		* linux agents:          -t "queueloc LIKE 'linux.%%'"
		* agents named *mysql*:  -t "name like '%%mysql%%'"
		* proxied linux agents:  -t "queueloc LIKE 'linux.%%' AND environment->>'isproxied' = 'true'"
		* agents operated by IT: -t "tags#>>'{operator}'='IT'"

--- Modules documentation ---
Each module provides its own set of parameters. Module parameters must be set *after*
global options for the parsing to work correctly. The following modules are available:
`, os.Args[0], os.Args[0])
	for module, _ := range mig.AvailableModules {
		fmt.Printf("* %s\n", module)
	}
	fmt.Printf("To access a module documentation, use: %s <module> help\n", os.Args[0])
	os.Exit(1)
}

func continueOnFlagError() {
	return
}

func main() {
	var (
		conf                                   client.Configuration
		cli                                    client.Client
		err                                    error
		op                                     mig.Operation
		a                                      mig.Action
		migrc, show, target, expiration, afile string
		modargs                                []string
		modRunner                              interface{}
	)
	defer func() {
		if e := recover(); e != nil {
			fmt.Fprintf(os.Stderr, "%v\n", e)
		}
	}()
	homedir := client.FindHomedir()
	fs := flag.NewFlagSet("mig flag", flag.ContinueOnError)
	fs.Usage = continueOnFlagError
	fs.StringVar(&migrc, "c", homedir+"/.migrc", "alternative configuration file")
	fs.StringVar(&show, "show", "found", "type of results to show")
	fs.StringVar(&target, "t", `status='online'`, "action target")
	fs.StringVar(&expiration, "e", "300s", "expiration")
	fs.StringVar(&afile, "i", "/path/to/file", "Load action from file")

	// if first argument is missing, or is help, print help
	// otherwise, pass the remainder of the arguments to the module for parsing
	// this client is agnostic to module parameters
	if len(os.Args) < 2 || os.Args[1] == "help" || os.Args[1] == "-h" || os.Args[1] == "--help" {
		usage()
	}

	// when reading the action from a file, go directly to launch
	if os.Args[1] == "-i" {
		err = fs.Parse(os.Args[1:])
		if err != nil {
			panic(err)
		}
		if afile == "/path/to/file" {
			panic("-i flag must take an action file path as argument")
		}
		a, err = mig.ActionFromFile(afile)
		if err != nil {
			panic(err)
		}
		fmt.Fprintf(os.Stderr, "[info] launching action from file, all flags are ignored")
		goto readytolaunch
	}

	// arguments parsing works as follow:
	// * os.Args[1] must contain the name of the module to launch. we first verify
	//   that a module exist for this name and then continue parsing
	// * os.Args[2:] contains both global options and module parameters. We parse the
	//   whole []string to extract global options, and module parameters will be left
	//   unparsed in fs.Args()
	// * fs.Args() with the module parameters is passed as a string to the module parser
	//   which will return a module operation to store in the action
	op.Module = os.Args[1]
	if _, ok := mig.AvailableModules[op.Module]; !ok {
		panic("Unknown module " + op.Module)
	}

	// -- Ugly hack Warning --
	// Parse() will fail on the first flag that is not defined, but in our case module flags
	// are defined in the module packages and not in this program. Therefore, the flag parse error
	// is expected. Unfortunately, Parse() writes directly to stderr and displays the error to
	// the user, which confuses them. The right fix would be to prevent Parse() from writing to
	// stderr, since that's really the job of the calling program, but in the meantime we work around
	// it by redirecting stderr to null before calling Parse(), and put it back to normal afterward.
	// for ref, issue is at https://github.com/golang/go/blob/master/src/flag/flag.go#L793
	fs.SetOutput(os.NewFile(uintptr(87592), os.DevNull))
	err = fs.Parse(os.Args[2:])
	fs.SetOutput(nil)
	if err != nil {
		// ignore the flag not defined error, which is expected because
		// module parameters are defined in modules and not in main
		if len(err.Error()) > 30 && err.Error()[0:29] == "flag provided but not defined" {
			// requeue the parameter that failed
			modargs = append(modargs, err.Error()[31:])
		} else {
			// if it's another error, panic
			panic(err)
		}
	}
	for _, arg := range fs.Args() {
		modargs = append(modargs, arg)
	}
	modRunner = mig.AvailableModules[op.Module]()
	if _, ok := modRunner.(mig.HasParamsParser); !ok {
		fmt.Fprintf(os.Stderr, "[error] module '%s' does not support command line invocation\n", op.Module)
		os.Exit(2)
	}
	op.Parameters, err = modRunner.(mig.HasParamsParser).ParamsParser(modargs)
	if err != nil || op.Parameters == nil {
		panic(err)
	}
	a.Operations = append(a.Operations, op)

	a.Name = op.Module + " on '" + target + "'"
	a.Target = target

readytolaunch:
	// instanciate an API client
	conf, err = client.ReadConfiguration(migrc)
	if err != nil {
		panic(err)
	}
	cli = client.NewClient(conf)

	// set the validity 60 second in the past to deal with clock skew
	a.ValidFrom = time.Now().Add(-60 * time.Second).UTC()
	period, err := time.ParseDuration(expiration)
	if err != nil {
		panic(err)
	}
	a.ExpireAfter = a.ValidFrom.Add(period)
	// add extra 60 seconds taken for clock skew
	a.ExpireAfter = a.ExpireAfter.Add(60 * time.Second).UTC()

	asig, err := cli.SignAction(a)
	if err != nil {
		panic(err)
	}
	a = asig

	// evaluate target before launch, give a change to cancel before going out to agents
	agents, err := cli.EvaluateAgentTarget(a.Target)
	if err != nil {
		panic(err)
	}
	fmt.Fprintf(os.Stderr, "%d agents will be targeted. ctrl+c to cancel. launching in ", len(agents))
	for i := 5; i > 0; i-- {
		time.Sleep(1 * time.Second)
		fmt.Fprintf(os.Stderr, "%d ", i)
	}
	fmt.Fprintf(os.Stderr, "GO\n")

	// launch and follow
	a, err = cli.PostAction(a)
	if err != nil {
		panic(err)
	}
	c := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		err = cli.FollowAction(a)
		if err != nil {
			panic(err)
		}
		done <- true
	}()
	select {
	case <-c:
		fmt.Fprintf(os.Stderr, "stop following action. agents may still be running. printing available results:\n")
		goto printresults
	case <-done:
		goto printresults
	}
printresults:
	err = cli.PrintActionResults(a, show)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package api

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"github.com/julienschmidt/httprouter"
	"github.com/proshik/githubstatbot/telegram"
	"io"
	"log"
	"net/http"
	"strings"
)

type AccessTokenReq struct {
	ClientId     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	Code         string `json:"code"`
}

type AccessTokenResp struct {
	AccessToken string `json:"access_token"`
	TokenType   string `json:"token_type"`
	Scope       string `json:"scope"`
}

var client = &http.Client{}

func (h *Handler) Index(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	http.ServeFile(w, r, h.staticPath+"/index.html")
}

func (h *Handler) Version(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	if checkAuth(w, r, h.basicAuth) {
		io.WriteString(w, "Version: 0.5.1")
		return
	}

	w.Header().Set("WWW-Authenticate", `Basic realm="MY REALM"`)
	w.WriteHeader(401)
	w.Write([]byte("401 Unauthorized\n"))
}

func checkAuth(w http.ResponseWriter, r *http.Request, ba *BasicAuth) bool {
	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 {
		return false
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false
	}

	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false
	}

	return pair[0] == ba.Username && pair[1] == ba.Password
}

func (h *Handler) GitHubRedirect(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	code := r.URL.Query().Get("code")
	if code == "" {
		log.Printf("Code is empty in response from github")
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}
	state := r.URL.Query().Get("state")
	if state == "" {
		log.Printf("State is empty in response from github")
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}

	//check state on valid and get chatId by state
	chatId, err := h.stateStore.Get(state)
	if err != nil {
		log.Printf("Not found chatId by state. Error: %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	//delete chatId value state store
	h.stateStore.Delete(state)

	//Build request for get accessToken
	bodyReq := AccessTokenReq{h.oAuth.ClientId, h.oAuth.ClientSecret, code}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(bodyReq)

	req, err := http.NewRequest("POST", "https://github.com/login/oauth/access_token", b)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")

	resp, err := client.Do(req)
	if err != nil {
		log.Printf("Erorr on build request object. Error: %v\n", err)
		h.bot.InformAuth(chatId, false)
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}

	defer resp.Body.Close()
	//decode response with accessToken
	var bodyResp AccessTokenResp
	json.NewDecoder(resp.Body).Decode(&bodyResp)

	//save token in storage
	h.tokenStore.Add(int64(chatId), bodyResp.AccessToken)
	//inform user in bot about success auth
	h.bot.InformAuth(chatId, true)
	//redirect user to bot page in telegram
	http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
}
<commit_msg>Update version in code.<commit_after>package api

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"github.com/julienschmidt/httprouter"
	"github.com/proshik/githubstatbot/telegram"
	"io"
	"log"
	"net/http"
	"strings"
	"fmt"
)

type AccessTokenReq struct {
	ClientId     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	Code         string `json:"code"`
}

type AccessTokenResp struct {
	AccessToken string `json:"access_token"`
	TokenType   string `json:"token_type"`
	Scope       string `json:"scope"`
}

var client = &http.Client{}

func (h *Handler) Index(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	http.ServeFile(w, r, h.staticPath+"/index.html")
}

func (h *Handler) Version(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	if checkAuth(w, r, h.basicAuth) {
		io.WriteString(w, "Version: 0.5.1")
		return
	}

	w.Header().Set("WWW-Authenticate", `Basic realm="MY REALM"`)
	w.WriteHeader(401)
	w.Write([]byte("401 Unauthorized\n"))
}

func checkAuth(w http.ResponseWriter, r *http.Request, ba *BasicAuth) bool {
	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 {
		return false
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false
	}

	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false
	}

	return pair[0] == ba.Username && pair[1] == ba.Password
}

func (h *Handler) GitHubRedirect(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	code := r.URL.Query().Get("code")
	if code == "" {
		log.Printf("Code is empty in response from github")
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}
	state := r.URL.Query().Get("state")
	if state == "" {
		log.Printf("State is empty in response from github")
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}

	//check state on valid and get chatId by state
	chatId, err := h.stateStore.Get(state)
	if err != nil {
		log.Printf("Not found chatId by state. Error: %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	//delete chatId value state store
	h.stateStore.Delete(state)

	//Build request for get accessToken
	bodyReq := AccessTokenReq{h.oAuth.ClientId, h.oAuth.ClientSecret, code}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(bodyReq)

	req, err := http.NewRequest("POST", "https://github.com/login/oauth/access_token", b)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")

	resp, err := client.Do(req)
	if err != nil {
		log.Printf("Erorr on build request object. Error: %v\n", err)
		h.bot.InformAuth(chatId, false)
		http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
		return
	}

	defer resp.Body.Close()
	//decode response with accessToken
	var bodyResp AccessTokenResp
	json.NewDecoder(resp.Body).Decode(&bodyResp)

	//save token in storage
	err = h.tokenStore.Add(int64(chatId), bodyResp.AccessToken)
	if err != nil {
		fmt.Println(err)
	}
	//inform user in bot about success auth
	h.bot.InformAuth(chatId, true)
	//redirect user to bot page in telegram
	//http.Redirect(w, r, telegram.RedirectBotAddress, http.StatusMovedPermanently)
}
<|endoftext|>