<commit_before>package device

import (
	"testing"

	"github.com/globocom/config"
	"launchpad.net/gocheck"

	"github.com/wiliamsouza/apollo/db"
)

func Test(t *testing.T) { gocheck.TestingT(t) }

type S struct{}

var _ = gocheck.Suite(&S{})

func (s *S) SetUpSuite(c *gocheck.C) {
	err := config.ReadConfigFile("../etc/apollod.conf")
	c.Check(err, gocheck.IsNil)
	config.Set("database:name", "apollo_customer_tests")
	db.Connect()
}

func (s *S) TearDownSuite(c *gocheck.C) {
	db.Session.DB.DropDatabase()
}

func (s *S) TestNewDevice(c *gocheck.C) {
	codename := "a700"
	permission := Permission{Run: true, Results: true, Info: true}
	permissions := Permissions{Organization: permission, Team: permission}
	d := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	device, err := NewDevice(d)
	c.Assert(err, gocheck.IsNil)
	defer db.Session.Device().RemoveId(codename)
	var deviceDb Device
	err = db.Session.Device().FindId(codename).One(&deviceDb)
	c.Assert(err, gocheck.IsNil)
	c.Assert(deviceDb, gocheck.DeepEquals, device)
}

func (s *S) TestListDevice(c *gocheck.C) {
	codename1 := "a700"
	codename2 := "a701"
	permission := Permission{Run: true, Results: true, Info: true}
	permissions := Permissions{Organization: permission, Team: permission}
	d1 := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	d2 := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a701",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	device1, err := NewDevice(d1)
	c.Assert(err, gocheck.IsNil)
	device2, err := NewDevice(d2)
	c.Assert(err, gocheck.IsNil)
	defer db.Session.Device().RemoveId(codename1)
	defer db.Session.Device().RemoveId(codename2)
	deviceList := DeviceList{device1, device2}
	deviceListDb, err := ListDevices()
	c.Assert(err, gocheck.IsNil)
	c.Assert(deviceListDb, gocheck.DeepEquals, deviceList)
}

func (s *S) TestDetailDevice(c *gocheck.C) {
	codename := "a700"
	permission := Permission{Run: true, Results: true, Info: true}
	permissions := Permissions{Organization: permission, Team: permission}
	d := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	device, err := NewDevice(d)
	c.Assert(err, gocheck.IsNil)
	defer db.Session.Device().RemoveId(codename)
	deviceDb, err := DetailDevice(codename)
	c.Assert(err, gocheck.IsNil)
	c.Assert(deviceDb, gocheck.DeepEquals, device)
}

func (s *S) TestModifyDevice(c *gocheck.C) {
	codename := "a700"
	permission := Permission{Run: true, Results: true, Info: true}
	permissions := Permissions{Organization: permission, Team: permission}
	d1 := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	d2 := Device{
		Permission:      permissions,
		Name:            "Acer A701",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 4",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "2GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "64GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "15MP",
		SecondaryCamera: "5MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	_, err := NewDevice(d1)
	c.Assert(err, gocheck.IsNil)
	defer db.Session.Device().RemoveId(codename)
	err = ModifyDevice(codename, d2)
	c.Assert(err, gocheck.IsNil)
	var deviceDb Device
	err = db.Session.Device().FindId(codename).One(&deviceDb)
	c.Assert(err, gocheck.IsNil)
	c.Assert(deviceDb, gocheck.DeepEquals, d2)
}

func (s *S) TestRemoveDevice(c *gocheck.C) {
	permission := Permission{Run: true, Results: true, Info: true}
	permissions := Permissions{Organization: permission, Team: permission}
	d := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	device, err := NewDevice(d)
	c.Assert(err, gocheck.IsNil)
	err = RemoveDevice(device.Codename)
	c.Assert(err, gocheck.IsNil)
	lenght, err := db.Session.Device().FindId(device.Codename).Count()
	c.Assert(err, gocheck.IsNil)
	c.Assert(lenght, gocheck.Equals, 0)
}
<commit_msg>Changed device test database name<commit_after>package device

import (
	"testing"

	"github.com/globocom/config"
	"launchpad.net/gocheck"

	"github.com/wiliamsouza/apollo/db"
)

func Test(t *testing.T) { gocheck.TestingT(t) }

type S struct{}

var _ = gocheck.Suite(&S{})

func (s *S) SetUpSuite(c *gocheck.C) {
	err := config.ReadConfigFile("../etc/apollod.conf")
	c.Check(err, gocheck.IsNil)
	config.Set("database:name", "apollo_device_tests")
	db.Connect()
}

func (s *S) TearDownSuite(c *gocheck.C) {
	db.Session.DB.DropDatabase()
}

func (s *S) TestNewDevice(c *gocheck.C) {
	codename := "a700"
	permission := Permission{Run: true, Results: true, Info: true}
	permissions := Permissions{Organization: permission, Team: permission}
	d := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	device, err := NewDevice(d)
	c.Assert(err, gocheck.IsNil)
	defer db.Session.Device().RemoveId(codename)
	var deviceDb Device
	err = db.Session.Device().FindId(codename).One(&deviceDb)
	c.Assert(err, gocheck.IsNil)
	c.Assert(deviceDb, gocheck.DeepEquals, device)
}

func (s *S) TestListDevice(c *gocheck.C) {
	codename1 := "a700"
	codename2 := "a701"
	permission := Permission{Run: true, Results: true, Info: true}
	permissions := Permissions{Organization: permission, Team: permission}
	d1 := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	d2 := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a701",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	device1, err := NewDevice(d1)
	c.Assert(err, gocheck.IsNil)
	device2, err := NewDevice(d2)
	c.Assert(err, gocheck.IsNil)
	defer db.Session.Device().RemoveId(codename1)
	defer db.Session.Device().RemoveId(codename2)
	deviceList := DeviceList{device1, device2}
	deviceListDb, err := ListDevices()
	c.Assert(err, gocheck.IsNil)
	c.Assert(deviceListDb, gocheck.DeepEquals, deviceList)
}

func (s *S) TestDetailDevice(c *gocheck.C) {
	codename := "a700"
	permission := Permission{Run: true, Results: true, Info: true}
	permissions := Permissions{Organization: permission, Team: permission}
	d := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	device, err := NewDevice(d)
	c.Assert(err, gocheck.IsNil)
	defer db.Session.Device().RemoveId(codename)
	deviceDb, err := DetailDevice(codename)
	c.Assert(err, gocheck.IsNil)
	c.Assert(deviceDb, gocheck.DeepEquals, device)
}

func (s *S) TestModifyDevice(c *gocheck.C) {
	codename := "a700"
	permission := Permission{Run: true, Results: true, Info: true}
	permissions := Permissions{Organization: permission, Team: permission}
	d1 := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	d2 := Device{
		Permission:      permissions,
		Name:            "Acer A701",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 4",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "2GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "64GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "15MP",
		SecondaryCamera: "5MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	_, err := NewDevice(d1)
	c.Assert(err, gocheck.IsNil)
	defer db.Session.Device().RemoveId(codename)
	err = ModifyDevice(codename, d2)
	c.Assert(err, gocheck.IsNil)
	var deviceDb Device
	err = db.Session.Device().FindId(codename).One(&deviceDb)
	c.Assert(err, gocheck.IsNil)
	c.Assert(deviceDb, gocheck.DeepEquals, d2)
}

func (s *S) TestRemoveDevice(c *gocheck.C) {
	permission := Permission{Run: true, Results: true, Info: true}
	permissions := Permissions{Organization: permission, Team: permission}
	d := Device{
		Permission:      permissions,
		Name:            "Acer A700",
		Codename:        "a700",
		Vendor:          "Acer",
		Manufacturer:    "acer",
		Type:            "tablet",
		Platform:        "NVIDIA Tegra 3",
		Cpu:             "1.3 GHz quad-core Cortex A9",
		Gpu:             "416 MHz twelve-core Nvidia GeForce ULP",
		Ram:             "1GB",
		Weight:          "665 g (1.47 lb)",
		Dimensions:      "259x175x11 mm (10.20x6.89x0.43 in)",
		ScreenDimension: "257 mm (10.1 in)",
		Resolution:      "1920x1200",
		ScreenDensity:   "224 PPI",
		InternalStorage: "32GB",
		SdCard:          "up to 32 GB",
		Bluetooth:       "yes",
		WiFi:            "802.11 b/g/n",
		MainCamera:      "5MP",
		SecondaryCamera: "2MP",
		Power:           "9800 mAh",
		Peripherals:     "accelerometer, gyroscope, proximity sensor, digital compass, GPS, magnometer, microphone",
	}
	device, err := NewDevice(d)
	c.Assert(err, gocheck.IsNil)
	err = RemoveDevice(device.Codename)
	c.Assert(err, gocheck.IsNil)
	lenght, err := db.Session.Device().FindId(device.Codename).Count()
	c.Assert(err, gocheck.IsNil)
	c.Assert(lenght, gocheck.Equals, 0)
}
<|endoftext|>


<commit_before>package api

import (
	"net/http"

	"github.com/asaskevich/govalidator"
	"github.com/bleenco/abstruse/pkg/lib"
	"github.com/bleenco/abstruse/pkg/render"
	"github.com/bleenco/abstruse/server/db/model"
	"github.com/bleenco/abstruse/server/db/repository"
	"go.uber.org/zap"
)

type providers struct {
	logger       *zap.SugaredLogger
	providerRepo repository.ProviderRepo
}

func newProviders(logger *zap.Logger) providers {
	return providers{
		logger:       logger.With(zap.String("api", "providers")).Sugar(),
		providerRepo: repository.NewProviderRepo(),
	}
}

func (p *providers) find() http.HandlerFunc {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		claims := claimsFromCtx(r.Context())

		providers, err := p.providerRepo.Find(claims.ID)
		if err != nil {
			render.JSON(w, http.StatusNotFound, render.Error{Message: err.Error()})
			return
		}

		render.JSON(w, http.StatusOK, providers)
	})
}

func (p *providers) create() http.HandlerFunc {
	type form struct {
		Name        string `json:"name" valid:"stringlength(5|12),required"`
		URL         string `json:"url" valid:"url,required"`
		AccessToken string `json:"accessToken" valid:"stringlength(12|50),required"`
		Secret      string `json:"secret" valid:"stringlength(5|50),required"`
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		claims := claimsFromCtx(r.Context())
		var f form
		var err error
		defer r.Body.Close()

		if err = lib.DecodeJSON(r.Body, &f); err != nil {
			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
			return
		}

		if valid, err := govalidator.ValidateStruct(f); err != nil || !valid {
			render.JSON(w, http.StatusBadRequest, render.Error{Message: err.Error()})
			return
		}

		provider := model.Provider{
			Name:        f.Name,
			URL:         f.URL,
			AccessToken: f.AccessToken,
			Secret:      f.Secret,
			UserID:      claims.ID,
		}

		provider, err = p.providerRepo.Create(provider)
		if err != nil {
			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
			return
		}

		render.JSON(w, http.StatusOK, provider)
	})
}

func (p *providers) sync() http.HandlerFunc {
	type form struct {
		ID uint `json:"id" valid:"required"`
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		claims := claimsFromCtx(r.Context())
		var f form
		var err error
		defer r.Body.Close()

		if err = lib.DecodeJSON(r.Body, &f); err != nil {
			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
			return
		}

		if valid, err := govalidator.ValidateStruct(f); err != nil || !valid {
			render.JSON(w, http.StatusBadRequest, render.Error{Message: err.Error()})
			return
		}

		if err := p.providerRepo.Sync(f.ID, claims.ID); err != nil {
			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
			return
		}

		render.JSON(w, http.StatusOK, render.Empty{})
	})
}

// func (p *providers) repos() http.HandlerFunc {
// 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// 		claims := claimsFromCtx(r.Context())
// 		providerID, err := strconv.Atoi(chi.URLParam(r, "id"))
// 		if err != nil {
// 			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
// 			return
// 		}
// 		page, err := strconv.Atoi(r.URL.Query().Get("page"))
// 		if err != nil {
// 			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
// 			return
// 		}
// 		size, err := strconv.Atoi(r.URL.Query().Get("size"))
// 		if err != nil {
// 			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
// 			return
// 		}

// 		repos, err := p.providerRepo.FindRepos(uint(providerID), claims.ID, page, size)
// 		if err != nil {
// 			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
// 			return
// 		}

// 		render.JSON(w, http.StatusOK, repos)
// 	})
// }
<commit_msg>cleanup(providers): cleanup unused providers endpoints<commit_after>package api

import (
	"net/http"

	"github.com/asaskevich/govalidator"
	"github.com/bleenco/abstruse/pkg/lib"
	"github.com/bleenco/abstruse/pkg/render"
	"github.com/bleenco/abstruse/server/db/model"
	"github.com/bleenco/abstruse/server/db/repository"
	"go.uber.org/zap"
)

type providers struct {
	logger       *zap.SugaredLogger
	providerRepo repository.ProviderRepo
}

func newProviders(logger *zap.Logger) providers {
	return providers{
		logger:       logger.With(zap.String("api", "providers")).Sugar(),
		providerRepo: repository.NewProviderRepo(),
	}
}

func (p *providers) find() http.HandlerFunc {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		claims := claimsFromCtx(r.Context())

		providers, err := p.providerRepo.Find(claims.ID)
		if err != nil {
			render.JSON(w, http.StatusNotFound, render.Error{Message: err.Error()})
			return
		}

		render.JSON(w, http.StatusOK, providers)
	})
}

func (p *providers) create() http.HandlerFunc {
	type form struct {
		Name        string `json:"name" valid:"stringlength(5|12),required"`
		URL         string `json:"url" valid:"url,required"`
		AccessToken string `json:"accessToken" valid:"stringlength(12|50),required"`
		Secret      string `json:"secret" valid:"stringlength(5|50),required"`
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		claims := claimsFromCtx(r.Context())
		var f form
		var err error
		defer r.Body.Close()

		if err = lib.DecodeJSON(r.Body, &f); err != nil {
			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
			return
		}

		if valid, err := govalidator.ValidateStruct(f); err != nil || !valid {
			render.JSON(w, http.StatusBadRequest, render.Error{Message: err.Error()})
			return
		}

		provider := model.Provider{
			Name:        f.Name,
			URL:         f.URL,
			AccessToken: f.AccessToken,
			Secret:      f.Secret,
			UserID:      claims.ID,
		}

		provider, err = p.providerRepo.Create(provider)
		if err != nil {
			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
			return
		}

		render.JSON(w, http.StatusOK, provider)
	})
}

func (p *providers) sync() http.HandlerFunc {
	type form struct {
		ID uint `json:"id" valid:"required"`
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		claims := claimsFromCtx(r.Context())
		var f form
		var err error
		defer r.Body.Close()

		if err = lib.DecodeJSON(r.Body, &f); err != nil {
			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
			return
		}

		if valid, err := govalidator.ValidateStruct(f); err != nil || !valid {
			render.JSON(w, http.StatusBadRequest, render.Error{Message: err.Error()})
			return
		}

		if err := p.providerRepo.Sync(f.ID, claims.ID); err != nil {
			render.JSON(w, http.StatusInternalServerError, render.Error{Message: err.Error()})
			return
		}

		render.JSON(w, http.StatusOK, render.Empty{})
	})
}
<|endoftext|>


<commit_before>package resources

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/kinesisanalytics"
)

type KinesisAnalyticsApplication struct {
	svc             *kinesisanalytics.KinesisAnalytics
	applicationName *string
}

func init() {
	register("KinesisAnalyticsApplication", ListKinesisAnalyticsApplications)
}

func ListKinesisAnalyticsApplications(sess *session.Session) ([]Resource, error) {
	svc := kinesisanalytics.New(sess)
	resources := []Resource{}
	var lastApplicationName *string
	params := &kinesisanalytics.ListApplicationsInput{
		Limit: aws.Int64(25),
	}

	for {
		output, err := svc.ListApplications(params)
		if err != nil {
			return nil, err
		}

		for _, applicationSummary := range output.ApplicationSummaries {
			resources = append(resources, &KinesisAnalyticsApplication{
				svc:             svc,
				applicationName: applicationSummary.ApplicationName,
			})
			lastApplicationName = applicationSummary.ApplicationName
		}

		if *output.HasMoreApplications == false {
			break
		}

		params.ExclusiveStartApplicationName = lastApplicationName
	}

	return resources, nil
}

func (f *KinesisAnalyticsApplication) Remove() error {

	output, err := f.svc.DescribeApplication(&kinesisanalytics.DescribeApplicationInput{
		ApplicationName: f.applicationName,
	})

	if err != nil {
		return err
	}
	createTimestamp := output.ApplicationDetail.CreateTimestamp

	_, err = f.svc.DeleteApplication(&kinesisanalytics.DeleteApplicationInput{
		ApplicationName: f.applicationName,
		CreateTimestamp: createTimestamp,
	})

	return err
}

func (f *KinesisAnalyticsApplication) String() string {
	return *f.applicationName
}
<commit_msg>Use kinesisanalyticsv2 to resolve: ERRO[0094] UnsupportedOperationException: xxxxxx was created/updated by kinesisanalyticsv2 SDK. Please use kinesisanalyticsv2 SDK to make changes to xxxxxx. (#687)<commit_after>package resources

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/kinesisanalyticsv2"
)

type KinesisAnalyticsApplication struct {
	svc             *kinesisanalyticsv2.KinesisAnalyticsV2
	applicationName *string
}

func init() {
	register("KinesisAnalyticsApplication", ListKinesisAnalyticsApplications)
}

func ListKinesisAnalyticsApplications(sess *session.Session) ([]Resource, error) {
	svc := kinesisanalyticsv2.New(sess)
	resources := []Resource{}
	params := &kinesisanalyticsv2.ListApplicationsInput{
		Limit: aws.Int64(25),
	}

	for {
		output, err := svc.ListApplications(params)
		if err != nil {
			return nil, err
		}

		for _, applicationSummary := range output.ApplicationSummaries {
			resources = append(resources, &KinesisAnalyticsApplication{
				svc:             svc,
				applicationName: applicationSummary.ApplicationName,
			})
		}

		if output.NextToken == nil {
			break
		}

		params.NextToken = output.NextToken
	}

	return resources, nil
}

func (f *KinesisAnalyticsApplication) Remove() error {

	output, err := f.svc.DescribeApplication(&kinesisanalyticsv2.DescribeApplicationInput{
		ApplicationName: f.applicationName,
	})

	if err != nil {
		return err
	}
	createTimestamp := output.ApplicationDetail.CreateTimestamp

	_, err = f.svc.DeleteApplication(&kinesisanalyticsv2.DeleteApplicationInput{
		ApplicationName: f.applicationName,
		CreateTimestamp: createTimestamp,
	})

	return err
}

func (f *KinesisAnalyticsApplication) String() string {
	return *f.applicationName
}
<|endoftext|>


<commit_before>package scamp

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"io"

	"strings"
	"sync"
	"sync/atomic"
)

type incomingMsgNo uint64
type outgoingMsgNo uint64

// Connection a scamp connection
type Connection struct {
	conn           *tls.Conn
	Fingerprint    string
	readWriter     *bufio.ReadWriter
	readWriterLock sync.Mutex
	incomingmsgno  incomingMsgNo
	outgoingmsgno  outgoingMsgNo
	pktToMsg       map[incomingMsgNo](*Message)
	msgs           chan *Message
	client         *Client
	isClosed       bool
	closedMutex    sync.Mutex
	scampDebugger  *scampDebugger
}

// DialConnection Used by Client to establish a secure connection to the remote service.
// TODO: You must use the *connection.Fingerprint to verify the
// remote host
func DialConnection(connspec string) (conn *Connection, err error) {
	// Trace.Printf("Dialing connection to `%s`", connspec)
	config := &tls.Config{
		InsecureSkipVerify: true,
	}
	config.BuildNameToCertificate()

	tlsConn, err := tls.Dial("tcp", connspec, config)
	if err != nil {
		return
	}
	// Trace.Printf("Past TLS")
	conn = NewConnection(tlsConn, "client")
	return
}

// NewConnection Used by Service
func NewConnection(tlsConn *tls.Conn, connType string) (conn *Connection) {
	conn = new(Connection)
	conn.conn = tlsConn

	// TODO get the end entity certificate instead
	peerCerts := conn.conn.ConnectionState().PeerCertificates
	if len(peerCerts) == 1 {
		peerCert := peerCerts[0]
		conn.Fingerprint = sha1FingerPrint(peerCert)
	}

	var reader io.Reader = conn.conn
	var writer io.Writer = conn.conn
	if enableWriteTee {
		var err error
		conn.scampDebugger, err = newScampDebugger(conn.conn, connType)
		if err != nil {
			panic(fmt.Sprintf("could not create debugger: %s", err))
		}
		// reader = conn.scampDebugger.WrapReader(reader)
		writer = io.MultiWriter(writer, conn.scampDebugger)
		debuggerReaderWriter := scampDebuggerReader{
			wraps: conn.scampDebugger,
		}
		reader = io.TeeReader(reader, &debuggerReaderWriter)
		// nothing
	}

	conn.readWriter = bufio.NewReadWriter(bufio.NewReader(reader), bufio.NewWriter(writer))
	conn.incomingmsgno = 0
	conn.outgoingmsgno = 0

	conn.pktToMsg = make(map[incomingMsgNo](*Message))
	conn.msgs = make(chan *Message)

	conn.isClosed = false
	go conn.packetReader()

	return
}

// SetClient sets the client for a *Connection
func (conn *Connection) SetClient(client *Client) {
	conn.client = client
}

func (conn *Connection) packetReader() (err error) {
	if conn == nil {
		return
	}
	// I think we only need to lock on writes, packetReader is only running
	// from one spot.
	// conn.readWriterLock.Lock()
	// defer conn.readWriterLock.Unlock()
	var pkt *Packet

PacketReaderLoop:
	for {
		// Trace.Printf("reading packet...")

		pkt, err = ReadPacket(conn.readWriter)
		if err != nil {
			// Warning.Printf("Client %v, packet reader go routine %v ReadPacket error %s\n", conn.client.ID, prNum, err)
			if strings.Contains(err.Error(), "readline error: EOF") {
				// Trace.Printf("%s", err)
			} else if strings.Contains(err.Error(), "use of closed network connection") {
				// Trace.Printf("%s", err)
			} else if strings.Contains(err.Error(), "connection reset by peer") {
				// Trace.Printf("%s", err)
			} else {
				// Trace.Printf("%s", err)
				Error.Printf("err: %s", err)
			}
			break PacketReaderLoop
		}

		err = conn.routePacket(pkt)
		if err != nil {
			// Trace.Printf("breaking PacketReaderLoop")
			break PacketReaderLoop
		}
	}

	close(conn.msgs)
	return
}

func (conn *Connection) routePacket(pkt *Packet) (err error) {
	var msg *Message
	// Trace.Printf("routing packet...")
	switch {
	case pkt.packetType == HEADER:
		// Trace.Printf("HEADER")

		incomingmsgno := atomic.LoadUint64((*uint64)(&conn.incomingmsgno))
		if pkt.msgNo != incomingmsgno {
			err = fmt.Errorf("out of sequence msgno: expected %d but got %d", incomingmsgno, pkt.msgNo)
			Error.Printf("%s", err)
			return err
		}

		msg = conn.pktToMsg[incomingMsgNo(pkt.msgNo)]
		if msg != nil {
			err = fmt.Errorf("Bad HEADER; already tracking msgno %d", pkt.msgNo)
			Error.Printf("%s", err)
			return err
		}

		// Allocate message and copy over header values so we don't have to track them
		// We copy out the packetHeader values and then we can discard it
		msg = NewMessage()
		msg.SetAction(pkt.packetHeader.Action)
		msg.SetEnvelope(pkt.packetHeader.Envelope)
		msg.SetVersion(pkt.packetHeader.Version)
		msg.SetMessageType(pkt.packetHeader.MessageType)
		msg.SetRequestID(pkt.packetHeader.RequestID)
		msg.SetError(pkt.packetHeader.Error)
		msg.SetErrorCode(pkt.packetHeader.ErrorCode)
		msg.SetTicket(pkt.packetHeader.Ticket)
		// TODO: Do we need the requestId?

		conn.pktToMsg[incomingMsgNo(pkt.msgNo)] = msg
		// This is for sending out data
		// conn.incomingNotifiers[pktMsgNo] = &make((chan *Message),1)

		atomic.AddUint64((*uint64)(&conn.incomingmsgno), 1)
	case pkt.packetType == DATA:
		// Trace.Printf("DATA")
		// Append data
		// Verify we are tracking that message
		msg = conn.pktToMsg[incomingMsgNo(pkt.msgNo)]
		if msg == nil {
			return fmt.Errorf("not tracking message number %d", pkt.msgNo)
		}

		msg.Write(pkt.body)
		conn.ackBytes(incomingMsgNo(pkt.msgNo), msg.BytesWritten())

	case pkt.packetType == EOF:
		// Trace.Printf("EOF")
		// Deliver message
		msg = conn.pktToMsg[incomingMsgNo(pkt.msgNo)]
		if msg == nil {
			err = fmt.Errorf("cannot process EOF for unknown msgno %d", pkt.msgNo)
			Error.Printf("err: `%s`", err)
			return
		}

		delete(conn.pktToMsg, incomingMsgNo(pkt.msgNo))
		// Trace.Printf("Delivering message number %d up the stack", pkt.msgNo)
		// Trace.Printf("Adding message to channel:")
		conn.msgs <- msg

	case pkt.packetType == TXERR:
		msg = conn.pktToMsg[incomingMsgNo(pkt.msgNo)]
		if msg == nil {
			err = fmt.Errorf("cannot process EOF for unknown msgno %d", pkt.msgNo)
			Error.Printf("err: `%s`", err)
			return
		}
		//get the error
		if len(pkt.body) > 0 {
			// Trace.Printf("getting error from packet body: %s", pkt.body)
			errMessage := string(pkt.body)
			msg.Error = errMessage
		} else {
			msg.Error = "There was an unkown error with the connection"
		}
		msg.Write(pkt.body)
		conn.ackBytes(incomingMsgNo(pkt.msgNo), msg.BytesWritten())

		delete(conn.pktToMsg, incomingMsgNo(pkt.msgNo))
		conn.msgs <- msg

	case pkt.packetType == ACK:
		// Trace.Printf("ACK `%v` for msgno %v", len(pkt.body), pkt.msgNo)
		// panic("Xavier needs to support this")
		// TODO: Add bytes to message stream tally
	}

	return
}

const RetryLimit = 50

// Send sends a scamp message using the current *Connection
func (conn *Connection) Send(msg *Message) (err error) {
	if conn.isClosed {
		err = fmt.Errorf("connection already closed")
	}

	conn.readWriterLock.Lock()
	defer conn.readWriterLock.Unlock()
	if msg.RequestID == 0 {
		err = fmt.Errorf("must specify `ReqestId` on msg before sending")
		return
	}

	outgoingmsgno := atomic.LoadUint64((*uint64)(&conn.outgoingmsgno))
	atomic.AddUint64((*uint64)(&conn.outgoingmsgno), 1)

	// Trace.Printf("sending msgno %d", outgoingmsgno)

	for _, pkt := range msg.toPackets(outgoingmsgno) {
		// Trace.Printf("sending pkt %d", i)

		retries := 0
		if enableWriteTee {
			writer := io.MultiWriter(conn.readWriter, conn.scampDebugger)
			_, err := pkt.Write(writer)
			conn.scampDebugger.file.Write([]byte("\n"))
			if err != nil {
				Error.Printf("error writing packet: `%s`", err)
				return err
			}
		} else {
			for {
				_, err := pkt.Write(conn.readWriter)
				// TODO: should we actually blacklist this error?
				if err != nil {
					//temprarily
					if strings.Contains(err.Error(), "use of closed connection") {
						err = fmt.Errorf("connection closed")
						break
					}

					if retries > RetryLimit {
						return fmt.Errorf("Retried too many times: %s", err)
					}

					Error.Printf("error writing packet: `%s` (retrying)", err)
					retries += 1
					continue
				}
				break
			}
		}

	}
	conn.readWriter.Flush()
	// Trace.Printf("done sending msg")

	return
}

func (conn *Connection) ackBytes(msgno incomingMsgNo, unackedByteCount uint64) (err error) {
	// Trace.Printf("ACKing msg %v, unacked bytes = %v", msgno, unackedByteCount)
	conn.readWriterLock.Lock()
	defer conn.readWriterLock.Unlock()

	ackPacket := Packet{
		packetType: ACK,
		msgNo:      uint64(msgno),
		body:       []byte(fmt.Sprintf("%d", unackedByteCount)),
	}

	var thisWriter io.Writer
	if enableWriteTee {
		thisWriter = io.MultiWriter(conn.readWriter, conn.scampDebugger)
	} else {
		thisWriter = conn.readWriter
	}

	_, err = ackPacket.Write(thisWriter)
	if err != nil {
		return err
	}

	conn.readWriter.Flush()

	return
}

// Close closes the current *Connection
func (conn *Connection) Close() {
	conn.closedMutex.Lock()
	if conn.isClosed {
		// Trace.Printf("connection already closed. skipping shutdown.")
		conn.closedMutex.Unlock()
		return
	}

	// Trace.Printf("connection is closing")

	conn.conn.Close()
	// conn.conn = nil

	// conn.readWriterLock.Lock()
	// conn.readWriter.Flush()
	// conn.readWriterLock.Unlock()

	conn.isClosed = true
	conn.closedMutex.Unlock()
}
<commit_msg>check for nil Connection (#12)<commit_after>package scamp

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"io"

	"strings"
	"sync"
	"sync/atomic"
)

type incomingMsgNo uint64
type outgoingMsgNo uint64

// Connection a scamp connection
type Connection struct {
	conn           *tls.Conn
	Fingerprint    string
	readWriter     *bufio.ReadWriter
	readWriterLock sync.Mutex
	incomingmsgno  incomingMsgNo
	outgoingmsgno  outgoingMsgNo
	pktToMsg       map[incomingMsgNo](*Message)
	msgs           chan *Message
	client         *Client
	isClosed       bool
	closedMutex    sync.Mutex
	scampDebugger  *scampDebugger
}

// DialConnection Used by Client to establish a secure connection to the remote service.
// TODO: You must use the *connection.Fingerprint to verify the
// remote host
func DialConnection(connspec string) (conn *Connection, err error) {
	// Trace.Printf("Dialing connection to `%s`", connspec)
	config := &tls.Config{
		InsecureSkipVerify: true,
	}
	config.BuildNameToCertificate()

	tlsConn, err := tls.Dial("tcp", connspec, config)
	if err != nil {
		return
	}
	// Trace.Printf("Past TLS")
	conn = NewConnection(tlsConn, "client")
	return
}

// NewConnection Used by Service
func NewConnection(tlsConn *tls.Conn, connType string) (conn *Connection) {
	conn = new(Connection)
	conn.conn = tlsConn

	// TODO get the end entity certificate instead
	peerCerts := conn.conn.ConnectionState().PeerCertificates
	if len(peerCerts) == 1 {
		peerCert := peerCerts[0]
		conn.Fingerprint = sha1FingerPrint(peerCert)
	}

	var reader io.Reader = conn.conn
	var writer io.Writer = conn.conn
	if enableWriteTee {
		var err error
		conn.scampDebugger, err = newScampDebugger(conn.conn, connType)
		if err != nil {
			panic(fmt.Sprintf("could not create debugger: %s", err))
		}
		// reader = conn.scampDebugger.WrapReader(reader)
		writer = io.MultiWriter(writer, conn.scampDebugger)
		debuggerReaderWriter := scampDebuggerReader{
			wraps: conn.scampDebugger,
		}
		reader = io.TeeReader(reader, &debuggerReaderWriter)
		// nothing
	}

	conn.readWriter = bufio.NewReadWriter(bufio.NewReader(reader), bufio.NewWriter(writer))
	conn.incomingmsgno = 0
	conn.outgoingmsgno = 0

	conn.pktToMsg = make(map[incomingMsgNo](*Message))
	conn.msgs = make(chan *Message)

	conn.isClosed = false
	go conn.packetReader()

	return
}

// SetClient sets the client for a *Connection
func (conn *Connection) SetClient(client *Client) {
	conn.client = client
}

func (conn *Connection) packetReader() (err error) {
	if conn == nil {
		return
	}
	// I think we only need to lock on writes, packetReader is only running
	// from one spot.
	// conn.readWriterLock.Lock()
	// defer conn.readWriterLock.Unlock()
	var pkt *Packet

PacketReaderLoop:
	for {
		// Trace.Printf("reading packet...")

		pkt, err = ReadPacket(conn.readWriter)
		if err != nil {
			// Warning.Printf("Client %v, packet reader go routine %v ReadPacket error %s\n", conn.client.ID, prNum, err)
			if strings.Contains(err.Error(), "readline error: EOF") {
				// Trace.Printf("%s", err)
			} else if strings.Contains(err.Error(), "use of closed network connection") {
				// Trace.Printf("%s", err)
			} else if strings.Contains(err.Error(), "connection reset by peer") {
				// Trace.Printf("%s", err)
			} else {
				// Trace.Printf("%s", err)
				Error.Printf("err: %s", err)
			}
			break PacketReaderLoop
		}

		err = conn.routePacket(pkt)
		if err != nil {
			// Trace.Printf("breaking PacketReaderLoop")
			break PacketReaderLoop
		}
	}

	close(conn.msgs)
	return
}

func (conn *Connection) routePacket(pkt *Packet) (err error) {
	var msg *Message
	// Trace.Printf("routing packet...")
	switch {
	case pkt.packetType == HEADER:
		// Trace.Printf("HEADER")

		incomingmsgno := atomic.LoadUint64((*uint64)(&conn.incomingmsgno))
		if pkt.msgNo != incomingmsgno {
			err = fmt.Errorf("out of sequence msgno: expected %d but got %d", incomingmsgno, pkt.msgNo)
			Error.Printf("%s", err)
			return err
		}

		msg = conn.pktToMsg[incomingMsgNo(pkt.msgNo)]
		if msg != nil {
			err = fmt.Errorf("Bad HEADER; already tracking msgno %d", pkt.msgNo)
			Error.Printf("%s", err)
			return err
		}

		// Allocate message and copy over header values so we don't have to track them
		// We copy out the packetHeader values and then we can discard it
		msg = NewMessage()
		msg.SetAction(pkt.packetHeader.Action)
		msg.SetEnvelope(pkt.packetHeader.Envelope)
		msg.SetVersion(pkt.packetHeader.Version)
		msg.SetMessageType(pkt.packetHeader.MessageType)
		msg.SetRequestID(pkt.packetHeader.RequestID)
		msg.SetError(pkt.packetHeader.Error)
		msg.SetErrorCode(pkt.packetHeader.ErrorCode)
		msg.SetTicket(pkt.packetHeader.Ticket)
		// TODO: Do we need the requestId?

		conn.pktToMsg[incomingMsgNo(pkt.msgNo)] = msg
		// This is for sending out data
		// conn.incomingNotifiers[pktMsgNo] = &make((chan *Message),1)

		atomic.AddUint64((*uint64)(&conn.incomingmsgno), 1)
	case pkt.packetType == DATA:
		// Trace.Printf("DATA")
		// Append data
		// Verify we are tracking that message
		msg = conn.pktToMsg[incomingMsgNo(pkt.msgNo)]
		if msg == nil {
			return fmt.Errorf("not tracking message number %d", pkt.msgNo)
		}

		msg.Write(pkt.body)
		conn.ackBytes(incomingMsgNo(pkt.msgNo), msg.BytesWritten())

	case pkt.packetType == EOF:
		// Trace.Printf("EOF")
		// Deliver message
		msg = conn.pktToMsg[incomingMsgNo(pkt.msgNo)]
		if msg == nil {
			err = fmt.Errorf("cannot process EOF for unknown msgno %d", pkt.msgNo)
			Error.Printf("err: `%s`", err)
			return
		}

		delete(conn.pktToMsg, incomingMsgNo(pkt.msgNo))
		// Trace.Printf("Delivering message number %d up the stack", pkt.msgNo)
		// Trace.Printf("Adding message to channel:")
		conn.msgs <- msg

	case pkt.packetType == TXERR:
		msg = conn.pktToMsg[incomingMsgNo(pkt.msgNo)]
		if msg == nil {
			err = fmt.Errorf("cannot process EOF for unknown msgno %d", pkt.msgNo)
			Error.Printf("err: `%s`", err)
			return
		}
		//get the error
		if len(pkt.body) > 0 {
			// Trace.Printf("getting error from packet body: %s", pkt.body)
			errMessage := string(pkt.body)
			msg.Error = errMessage
		} else {
			msg.Error = "There was an unkown error with the connection"
		}
		msg.Write(pkt.body)
		conn.ackBytes(incomingMsgNo(pkt.msgNo), msg.BytesWritten())

		delete(conn.pktToMsg, incomingMsgNo(pkt.msgNo))
		conn.msgs <- msg

	case pkt.packetType == ACK:
		// Trace.Printf("ACK `%v` for msgno %v", len(pkt.body), pkt.msgNo)
		// panic("Xavier needs to support this")
		// TODO: Add bytes to message stream tally
	}

	return
}

const RetryLimit = 50

// Send sends a scamp message using the current *Connection
func (conn *Connection) Send(msg *Message) (err error) {\
	if conn == nil {
		return fmt.Errorf("cannot send on nil connection")
	}
	if conn.isClosed {
		err = fmt.Errorf("connection already closed")
	}

	conn.readWriterLock.Lock()
	defer conn.readWriterLock.Unlock()
	if msg.RequestID == 0 {
		err = fmt.Errorf("must specify `ReqestId` on msg before sending")
		return
	}

	outgoingmsgno := atomic.LoadUint64((*uint64)(&conn.outgoingmsgno))
	atomic.AddUint64((*uint64)(&conn.outgoingmsgno), 1)

	// Trace.Printf("sending msgno %d", outgoingmsgno)

	for _, pkt := range msg.toPackets(outgoingmsgno) {
		// Trace.Printf("sending pkt %d", i)

		retries := 0
		if enableWriteTee {
			writer := io.MultiWriter(conn.readWriter, conn.scampDebugger)
			_, err := pkt.Write(writer)
			conn.scampDebugger.file.Write([]byte("\n"))
			if err != nil {
				Error.Printf("error writing packet: `%s`", err)
				return err
			}
		} else {
			for {
				_, err := pkt.Write(conn.readWriter)
				// TODO: should we actually blacklist this error?
				if err != nil {
					//temprarily
					if strings.Contains(err.Error(), "use of closed connection") {
						err = fmt.Errorf("connection closed")
						break
					}

					if retries > RetryLimit {
						return fmt.Errorf("Retried too many times: %s", err)
					}

					Error.Printf("error writing packet: `%s` (retrying)", err)
					retries += 1
					continue
				}
				break
			}
		}

	}
	conn.readWriter.Flush()
	// Trace.Printf("done sending msg")

	return
}

func (conn *Connection) ackBytes(msgno incomingMsgNo, unackedByteCount uint64) (err error) {
	// Trace.Printf("ACKing msg %v, unacked bytes = %v", msgno, unackedByteCount)
	conn.readWriterLock.Lock()
	defer conn.readWriterLock.Unlock()

	ackPacket := Packet{
		packetType: ACK,
		msgNo:      uint64(msgno),
		body:       []byte(fmt.Sprintf("%d", unackedByteCount)),
	}

	var thisWriter io.Writer
	if enableWriteTee {
		thisWriter = io.MultiWriter(conn.readWriter, conn.scampDebugger)
	} else {
		thisWriter = conn.readWriter
	}

	_, err = ackPacket.Write(thisWriter)
	if err != nil {
		return err
	}

	conn.readWriter.Flush()

	return
}

// Close closes the current *Connection
func (conn *Connection) Close() {
	conn.closedMutex.Lock()
	if conn.isClosed {
		// Trace.Printf("connection already closed. skipping shutdown.")
		conn.closedMutex.Unlock()
		return
	}

	// Trace.Printf("connection is closing")

	conn.conn.Close()
	// conn.conn = nil

	// conn.readWriterLock.Lock()
	// conn.readWriter.Flush()
	// conn.readWriterLock.Unlock()

	conn.isClosed = true
	conn.closedMutex.Unlock()
}
<|endoftext|>


<commit_before>package api

import (
	"context"
	"fmt"
	"github.com/Aptomi/aptomi/pkg/lang"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/dgrijalva/jwt-go"
	jwtreq "github.com/dgrijalva/jwt-go/request"
	"github.com/julienschmidt/httprouter"
	"net/http"
	"time"
)

// AuthSuccessObject contains Info for the AuthSuccess type
var AuthSuccessObject = &runtime.Info{
	Kind:        "auth-success",
	Constructor: func() runtime.Object { return &AuthSuccess{} },
}

// AuthSuccess represents successful authentication
type AuthSuccess struct {
	runtime.TypeKind `yaml:",inline"`
	Token            string
}

func (api *coreAPI) handleLogin(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
	username := request.PostFormValue("username")
	password := request.PostFormValue("password")
	user, err := api.externalData.UserLoader.Authenticate(username, password)
	if err != nil {
		serverErr := NewServerError(fmt.Sprintf("Authentication error: %s", err))
		api.contentType.WriteOne(writer, request, serverErr)
	} else {
		api.contentType.WriteOne(writer, request, &AuthSuccess{
			AuthSuccessObject.GetTypeKind(),
			api.newToken(user),
		})
	}
}

type Claims struct {
	Name        string `json:"name"`
	DomainAdmin bool   `json:"admin,omitempty"`
	jwt.StandardClaims
}

func (claims Claims) Valid() error {
	if len(claims.Name) == 0 {
		return fmt.Errorf("token should contain non-empty username")
	}

	return claims.StandardClaims.Valid()
}

func (api *coreAPI) newToken(user *lang.User) string {
	token := jwt.NewWithClaims(jwt.SigningMethodHS256, Claims{
		Name:        user.Name,
		DomainAdmin: user.DomainAdmin,
		StandardClaims: jwt.StandardClaims{
			IssuedAt:  time.Now().Unix(),
			ExpiresAt: time.Now().Add(30 * 24 * time.Hour).Unix(),
		},
	})

	// Sign and get the complete encoded token as a string using the secret
	tokenString, err := token.SignedString([]byte(api.secret))
	if err != nil {
		panic(fmt.Errorf("error while signing token: %s", err))
	}

	return tokenString
}

func (api *coreAPI) auth(handle httprouter.Handle) httprouter.Handle {
	return api.handleAuth(handle, false)
}

func (api *coreAPI) admin(handle httprouter.Handle) httprouter.Handle {
	return api.handleAuth(handle, true)
}

func (api *coreAPI) handleAuth(handle httprouter.Handle, admin bool) httprouter.Handle {
	return func(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
		err := api.checkToken(request, admin)
		if err != nil {
			authErr := NewServerError(fmt.Sprintf("Authentication error: %s", err))
			api.contentType.WriteOneWithStatus(writer, request, authErr, http.StatusUnauthorized)
			return
		}

		handle(writer, request, params)
	}
}

const (
	ctxUserProperty = "user"
)

func (api *coreAPI) checkToken(request *http.Request, admin bool) error {
	token, err := jwtreq.ParseFromRequestWithClaims(request, jwtreq.AuthorizationHeaderExtractor, &Claims{},
		func(token *jwt.Token) (interface{}, error) {
			return []byte(api.secret), nil
		})
	if err != nil {
		return err
	}
	if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
		return fmt.Errorf("unexpected token signing method: %s", token.Header["alg"])
	}

	claims := token.Claims.(*Claims)
	user := api.externalData.UserLoader.LoadUserByName(claims.Name)
	if user == nil {
		return fmt.Errorf("token refers to non-existing user: %s", claims.Name)
	}
	if user.DomainAdmin != claims.DomainAdmin {
		return fmt.Errorf("token contains incorrect admin status: %t", claims.DomainAdmin)
	}

	if admin && !user.DomainAdmin {
		return fmt.Errorf("admin privileges required")
	}

	// store user into the request
	newRequest := request.WithContext(context.WithValue(request.Context(), ctxUserProperty, user))
	*request = *newRequest

	return nil
}

func (api *coreAPI) getUserOptional(request *http.Request) *lang.User {
	val := request.Context().Value(ctxUserProperty)
	if val == nil {
		return nil
	}
	if user, ok := val.(*lang.User); ok {
		return user
	}

	return nil
}

func (api *coreAPI) getUserRequired(request *http.Request) *lang.User {
	user := api.getUserOptional(request)
	if user == nil {
		panic("unauthorized or user couldn't be loaded")
	}

	return user
}
<commit_msg>Fix linter and avoid context key collisions<commit_after>package api

import (
	"context"
	"fmt"
	"github.com/Aptomi/aptomi/pkg/lang"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/dgrijalva/jwt-go"
	jwtreq "github.com/dgrijalva/jwt-go/request"
	"github.com/julienschmidt/httprouter"
	"net/http"
	"time"
)

// AuthSuccessObject contains Info for the AuthSuccess type
var AuthSuccessObject = &runtime.Info{
	Kind:        "auth-success",
	Constructor: func() runtime.Object { return &AuthSuccess{} },
}

// AuthSuccess represents successful authentication
type AuthSuccess struct {
	runtime.TypeKind `yaml:",inline"`
	Token            string
}

func (api *coreAPI) handleLogin(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
	username := request.PostFormValue("username")
	password := request.PostFormValue("password")
	user, err := api.externalData.UserLoader.Authenticate(username, password)
	if err != nil {
		serverErr := NewServerError(fmt.Sprintf("Authentication error: %s", err))
		api.contentType.WriteOne(writer, request, serverErr)
	} else {
		api.contentType.WriteOne(writer, request, &AuthSuccess{
			AuthSuccessObject.GetTypeKind(),
			api.newToken(user),
		})
	}
}

// Claims represent Aptomi JWT Claims
type Claims struct {
	Name        string `json:"name"`
	DomainAdmin bool   `json:"admin,omitempty"`
	jwt.StandardClaims
}

// Valid checks if claims are valid
func (claims Claims) Valid() error {
	if len(claims.Name) == 0 {
		return fmt.Errorf("token should contain non-empty username")
	}

	return claims.StandardClaims.Valid()
}

func (api *coreAPI) newToken(user *lang.User) string {
	token := jwt.NewWithClaims(jwt.SigningMethodHS256, Claims{
		Name:        user.Name,
		DomainAdmin: user.DomainAdmin,
		StandardClaims: jwt.StandardClaims{
			IssuedAt:  time.Now().Unix(),
			ExpiresAt: time.Now().Add(30 * 24 * time.Hour).Unix(),
		},
	})

	// Sign and get the complete encoded token as a string using the secret
	tokenString, err := token.SignedString([]byte(api.secret))
	if err != nil {
		panic(fmt.Errorf("error while signing token: %s", err))
	}

	return tokenString
}

func (api *coreAPI) auth(handle httprouter.Handle) httprouter.Handle {
	return api.handleAuth(handle, false)
}

func (api *coreAPI) admin(handle httprouter.Handle) httprouter.Handle {
	return api.handleAuth(handle, true)
}

func (api *coreAPI) handleAuth(handle httprouter.Handle, admin bool) httprouter.Handle {
	return func(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
		err := api.checkToken(request, admin)
		if err != nil {
			authErr := NewServerError(fmt.Sprintf("Authentication error: %s", err))
			api.contentType.WriteOneWithStatus(writer, request, authErr, http.StatusUnauthorized)
			return
		}

		handle(writer, request, params)
	}
}

// The key type is unexported to prevent collisions with context keys defined in other packages
type key int

const (
	// ctxUserKey is the context key for user
	ctxUserKey key = iota
)

func (api *coreAPI) checkToken(request *http.Request, admin bool) error {
	token, err := jwtreq.ParseFromRequestWithClaims(request, jwtreq.AuthorizationHeaderExtractor, &Claims{},
		func(token *jwt.Token) (interface{}, error) {
			return []byte(api.secret), nil
		})
	if err != nil {
		return err
	}
	if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
		return fmt.Errorf("unexpected token signing method: %s", token.Header["alg"])
	}

	claims := token.Claims.(*Claims)
	user := api.externalData.UserLoader.LoadUserByName(claims.Name)
	if user == nil {
		return fmt.Errorf("token refers to non-existing user: %s", claims.Name)
	}
	if user.DomainAdmin != claims.DomainAdmin {
		return fmt.Errorf("token contains incorrect admin status: %t", claims.DomainAdmin)
	}

	if admin && !user.DomainAdmin {
		return fmt.Errorf("admin privileges required")
	}

	// store user into the request
	newRequest := request.WithContext(context.WithValue(request.Context(), ctxUserKey, user))
	*request = *newRequest

	return nil
}

func (api *coreAPI) getUserOptional(request *http.Request) *lang.User {
	val := request.Context().Value(ctxUserKey)
	if val == nil {
		return nil
	}
	if user, ok := val.(*lang.User); ok {
		return user
	}

	return nil
}

func (api *coreAPI) getUserRequired(request *http.Request) *lang.User {
	user := api.getUserOptional(request)
	if user == nil {
		panic("unauthorized or user couldn't be loaded")
	}

	return user
}
<|endoftext|>


<commit_before>package apps

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	git "github.com/cozy/go-git"
	gitPlumbing "github.com/cozy/go-git/plumbing"
	gitObject "github.com/cozy/go-git/plumbing/object"
	gitStorage "github.com/cozy/go-git/storage/filesystem"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	gitOsFS "gopkg.in/src-d/go-billy.v2/osfs"
)

var errCloneTimeout = errors.New("git: repository cloning timed out")
var cloneTimeout = 20 * time.Second

const (
	ghRawManifestURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
	glRawManifestURL = "https://%s/%s/%s/raw/%s/%s"
)

var (
	// ghURLRegex is used to identify github
	ghURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
	// glURLRegex is used to identify gitlab
	glURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
)

type gitFetcher struct {
	manFilename string
	log         *logrus.Entry
}

func newGitFetcher(appType AppType, log *logrus.Entry) *gitFetcher {
	var manFilename string
	switch appType {
	case Webapp:
		manFilename = WebappManifestName
	case Konnector:
		manFilename = KonnectorManifestName
	}
	return &gitFetcher{
		manFilename: manFilename,
		log:         log,
	}
}

var manifestClient = &http.Client{
	Timeout: 60 * time.Second,
}

func isGithub(src *url.URL) bool {
	return src.Host == "github.com"
}

func isGitlab(src *url.URL) bool {
	return src.Host == "framagit.org" || strings.Contains(src.Host, "gitlab")
}

func (g *gitFetcher) FetchManifest(src *url.URL) (r io.ReadCloser, err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("[git] Error while fetching app manifest %s: %s",
				src.String(), err.Error())
		}
	}()

	if isGitSSHScheme(src.Scheme) {
		return g.fetchManifestFromGitArchive(src)
	}

	var u string
	if isGithub(src) {
		u, err = resolveGithubURL(src, g.manFilename)
	} else if isGitlab(src) {
		u, err = resolveGitlabURL(src, g.manFilename)
	} else {
		u, err = resolveManifestURL(src, g.manFilename)
	}
	if err != nil {
		return nil, err
	}

	g.log.Infof("[git] Fetching manifest on %s", u)
	res, err := manifestClient.Get(u)
	if err != nil || res.StatusCode != 200 {
		g.log.Errorf("[git] Error while fetching manifest on %s", u)
		return nil, ErrManifestNotReachable
	}

	return res.Body, nil
}

// Use the git archive method to download a manifest from the git repository.
func (g *gitFetcher) fetchManifestFromGitArchive(src *url.URL) (io.ReadCloser, error) {
	var branch string
	src, branch = getRemoteURL(src)
	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, "git",
		"archive",
		"--remote", src.String(),
		fmt.Sprintf("refs/heads/%s", branch),
		g.manFilename) // #nosec
	stdout, err := cmd.CombinedOutput()
	if err != nil {
		if err == exec.ErrNotFound {
			return nil, ErrNotSupportedSource
		}
		return nil, ErrManifestNotReachable
	}
	buf := new(bytes.Buffer)
	r := tar.NewReader(bytes.NewReader(stdout))
	for {
		h, err := r.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, ErrManifestNotReachable
		}
		if h.Name != g.manFilename {
			continue
		}
		if _, err = io.Copy(buf, r); err != nil {
			return nil, ErrManifestNotReachable
		}
		return ioutil.NopCloser(buf), nil
	}
	return nil, ErrManifestNotReachable
}

func (g *gitFetcher) Fetch(src *url.URL, fs Copier, man Manifest) (err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("[git] Error while fetching or copying repository %s: %s",
				src.String(), err.Error())
		}
	}()

	osFs := afero.NewOsFs()
	gitDir, err := afero.TempDir(osFs, "", "cozy-app-"+man.Slug())
	if err != nil {
		return err
	}
	defer osFs.RemoveAll(gitDir)

	gitFs := afero.NewBasePathFs(osFs, gitDir)
	// XXX Gitlab doesn't support the git protocol
	if src.Scheme == "git" && isGitlab(src) {
		src.Scheme = "https"
	}

	// If the scheme uses ssh, we have to use the git command.
	if isGitSSHScheme(src.Scheme) {
		err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
		if err == exec.ErrNotFound {
			return ErrNotSupportedSource
		}
		return err
	}

	err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
	if err != exec.ErrNotFound {
		return err
	}

	return g.fetchWithGoGit(gitDir, src, fs, man)
}

func (g *gitFetcher) fetchWithGit(gitFs afero.Fs, gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)
	srcStr := src.String()

	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()

	// The first command we execute is a ls-remote to check the last commit from
	// the remote branch and see if we already have a checked-out version of this
	// tree.
	cmd := exec.CommandContext(ctx, "git",
		"ls-remote", "--quiet",
		srcStr, fmt.Sprintf("refs/heads/%s", branch)) // #nosec
	lsRemote, err := cmd.CombinedOutput()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("[git] ls-remote error of %s %s: %s", srcStr, err.Error(),
				lsRemote)
		}
		return err
	}

	lsRemoteFields := bytes.Fields(lsRemote)
	if len(lsRemoteFields) == 0 {
		return fmt.Errorf("git: unexpected ls-remote output")
	}

	slug := man.Slug()
	version := man.Version() + "-" + string(lsRemoteFields[0])

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil {
		return err
	}
	defer func() {
		if errc := fs.Close(); errc != nil {
			err = errc
		}
	}()
	if exists {
		return nil
	}

	cmd = exec.CommandContext(ctx, "git",
		"clone",
		"--quiet",
		"--depth", "1",
		"--single-branch",
		"--branch", branch,
		"--", srcStr, gitDir) // #nosec

	g.log.Infof("[git] Clone with git %s %s in %s: %s", srcStr, branch, gitDir,
		strings.Join(cmd.Args, " "))
	stdoutStderr, err := cmd.CombinedOutput()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("[git] Clone error of %s %s: %s", srcStr, stdoutStderr,
				err.Error())
		}
		return err
	}

	return afero.Walk(gitFs, "/", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if info.Name() == ".git" {
				return filepath.SkipDir
			}
			return nil
		}
		src, err := gitFs.Open(path)
		if err != nil {
			return err
		}
		return fs.Copy(&fileInfo{
			name: path,
			size: info.Size(),
			mode: info.Mode(),
		}, src)
	})
}

func (g *gitFetcher) fetchWithGoGit(gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)

	storage, err := gitStorage.NewStorage(gitOsFS.New(gitDir))
	if err != nil {
		return err
	}

	errch := make(chan error)
	repch := make(chan *git.Repository)

	srcStr := src.String()
	g.log.Infof("[git] Clone with go-git %s %s in %s", srcStr, branch, gitDir)
	go func() {
		repc, errc := git.Clone(storage, nil, &git.CloneOptions{
			URL:           srcStr,
			Depth:         1,
			SingleBranch:  true,
			ReferenceName: gitPlumbing.ReferenceName(branch),
		})
		if errc != nil {
			errch <- errc
		} else {
			repch <- repc
		}
	}()

	var rep *git.Repository
	select {
	case rep = <-repch:
	case err = <-errch:
		g.log.Errorf("[git] Clone error of %s: %s", srcStr, err.Error())
		return err
	case <-time.After(cloneTimeout):
		g.log.Errorf("[git] Clone timeout of %s", srcStr)
		return errCloneTimeout
	}

	ref, err := rep.Head()
	if err != nil {
		return err
	}

	slug := man.Slug()
	version := man.Version() + "-" + ref.Hash().String()

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil {
		return err
	}
	defer func() {
		if errc := fs.Close(); errc != nil {
			err = errc
		}
	}()
	if exists {
		return nil
	}

	commit, err := rep.CommitObject(ref.Hash())
	if err != nil {
		return err
	}

	files, err := commit.Files()
	if err != nil {
		return err
	}

	return files.ForEach(func(f *gitObject.File) error {
		var r io.ReadCloser
		r, err = f.Reader()
		if err != nil {
			return err
		}
		defer r.Close()
		return fs.Copy(&fileInfo{
			name: f.Name,
			size: f.Size,
			mode: os.FileMode(f.Mode),
		}, r)
	})
}

func getWebBranch(src *url.URL) string {
	if src.Fragment != "" {
		return src.Fragment
	}
	return "HEAD"
}

func getRemoteURL(src *url.URL) (*url.URL, string) {
	branch := src.Fragment
	if branch == "" {
		branch = "master"
	}
	clonedSrc := *src
	clonedSrc.Fragment = ""
	return &clonedSrc, branch
}

func resolveGithubURL(src *url.URL, filename string) (string, error) {
	match := ghURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(ghRawManifestURL, user, project, branch, filename)
	return u, nil
}

func resolveGitlabURL(src *url.URL, filename string) (string, error) {
	match := glURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(glRawManifestURL, src.Host, user, project, branch, filename)
	return u, nil
}

func resolveManifestURL(src *url.URL, filename string) (string, error) {
	// TODO check that it works with a branch
	srccopy, _ := url.Parse(src.String())
	srccopy.Scheme = "http"
	if srccopy.Path == "" || srccopy.Path[len(srccopy.Path)-1] != '/' {
		srccopy.Path += "/"
	}
	srccopy.Path = srccopy.Path + filename
	return srccopy.String(), nil
}

func isGitSSHScheme(scheme string) bool {
	return scheme == "git+ssh" || scheme == "ssh+git"
}

var (
	_ Fetcher = &gitFetcher{}
)
<commit_msg>Fix archive cmd and only get the stdout<commit_after>package apps

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	git "github.com/cozy/go-git"
	gitPlumbing "github.com/cozy/go-git/plumbing"
	gitObject "github.com/cozy/go-git/plumbing/object"
	gitStorage "github.com/cozy/go-git/storage/filesystem"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	gitOsFS "gopkg.in/src-d/go-billy.v2/osfs"
)

var errCloneTimeout = errors.New("git: repository cloning timed out")
var cloneTimeout = 20 * time.Second

const (
	ghRawManifestURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
	glRawManifestURL = "https://%s/%s/%s/raw/%s/%s"
)

var (
	// ghURLRegex is used to identify github
	ghURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
	// glURLRegex is used to identify gitlab
	glURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
)

type gitFetcher struct {
	manFilename string
	log         *logrus.Entry
}

func newGitFetcher(appType AppType, log *logrus.Entry) *gitFetcher {
	var manFilename string
	switch appType {
	case Webapp:
		manFilename = WebappManifestName
	case Konnector:
		manFilename = KonnectorManifestName
	}
	return &gitFetcher{
		manFilename: manFilename,
		log:         log,
	}
}

var manifestClient = &http.Client{
	Timeout: 60 * time.Second,
}

func isGithub(src *url.URL) bool {
	return src.Host == "github.com"
}

func isGitlab(src *url.URL) bool {
	return src.Host == "framagit.org" || strings.Contains(src.Host, "gitlab")
}

func (g *gitFetcher) FetchManifest(src *url.URL) (r io.ReadCloser, err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("[git] Error while fetching app manifest %s: %s",
				src.String(), err.Error())
		}
	}()

	if isGitSSHScheme(src.Scheme) {
		return g.fetchManifestFromGitArchive(src)
	}

	var u string
	if isGithub(src) {
		u, err = resolveGithubURL(src, g.manFilename)
	} else if isGitlab(src) {
		u, err = resolveGitlabURL(src, g.manFilename)
	} else {
		u, err = resolveManifestURL(src, g.manFilename)
	}
	if err != nil {
		return nil, err
	}

	g.log.Infof("[git] Fetching manifest on %s", u)
	res, err := manifestClient.Get(u)
	if err != nil || res.StatusCode != 200 {
		g.log.Errorf("[git] Error while fetching manifest on %s", u)
		return nil, ErrManifestNotReachable
	}

	return res.Body, nil
}

// Use the git archive method to download a manifest from the git repository.
func (g *gitFetcher) fetchManifestFromGitArchive(src *url.URL) (io.ReadCloser, error) {
	var branch string
	src, branch = getRemoteURL(src)
	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, "git",
		"archive",
		"--remote", src.String(),
		fmt.Sprintf("refs/heads/%s", branch),
		g.manFilename) // #nosec
	g.log.Infof("[git] Fetching manifest %s", strings.Join(cmd.Args, " "))
	stdout, err := cmd.Output()
	if err != nil {
		if err == exec.ErrNotFound {
			return nil, ErrNotSupportedSource
		}
		return nil, ErrManifestNotReachable
	}
	buf := new(bytes.Buffer)
	r := tar.NewReader(bytes.NewReader(stdout))
	for {
		h, err := r.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, ErrManifestNotReachable
		}
		if h.Name != g.manFilename {
			continue
		}
		if _, err = io.Copy(buf, r); err != nil {
			return nil, ErrManifestNotReachable
		}
		return ioutil.NopCloser(buf), nil
	}
	return nil, ErrManifestNotReachable
}

func (g *gitFetcher) Fetch(src *url.URL, fs Copier, man Manifest) (err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("[git] Error while fetching or copying repository %s: %s",
				src.String(), err.Error())
		}
	}()

	osFs := afero.NewOsFs()
	gitDir, err := afero.TempDir(osFs, "", "cozy-app-"+man.Slug())
	if err != nil {
		return err
	}
	defer osFs.RemoveAll(gitDir)

	gitFs := afero.NewBasePathFs(osFs, gitDir)
	// XXX Gitlab doesn't support the git protocol
	if src.Scheme == "git" && isGitlab(src) {
		src.Scheme = "https"
	}

	// If the scheme uses ssh, we have to use the git command.
	if isGitSSHScheme(src.Scheme) {
		err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
		if err == exec.ErrNotFound {
			return ErrNotSupportedSource
		}
		return err
	}

	err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
	if err != exec.ErrNotFound {
		return err
	}

	return g.fetchWithGoGit(gitDir, src, fs, man)
}

func (g *gitFetcher) fetchWithGit(gitFs afero.Fs, gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)
	srcStr := src.String()

	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()

	// The first command we execute is a ls-remote to check the last commit from
	// the remote branch and see if we already have a checked-out version of this
	// tree.
	cmd := exec.CommandContext(ctx, "git",
		"ls-remote", "--quiet",
		srcStr, fmt.Sprintf("refs/heads/%s", branch)) // #nosec
	lsRemote, err := cmd.Output()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("[git] ls-remote error of %s: %s",
				strings.Join(cmd.Args, " "), err.Error())
		}
		return err
	}

	lsRemoteFields := bytes.Fields(lsRemote)
	if len(lsRemoteFields) == 0 {
		return fmt.Errorf("git: unexpected ls-remote output")
	}

	slug := man.Slug()
	version := man.Version() + "-" + string(lsRemoteFields[0])

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil {
		return err
	}
	defer func() {
		if errc := fs.Close(); errc != nil {
			err = errc
		}
	}()
	if exists {
		return nil
	}

	cmd = exec.CommandContext(ctx, "git",
		"clone",
		"--quiet",
		"--depth", "1",
		"--single-branch",
		"--branch", branch,
		"--", srcStr, gitDir) // #nosec

	g.log.Infof("[git] Clone with git %s %s in %s: %s", srcStr, branch, gitDir,
		strings.Join(cmd.Args, " "))
	stdoutStderr, err := cmd.CombinedOutput()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("[git] Clone error of %s %s: %s", srcStr, stdoutStderr,
				err.Error())
		}
		return err
	}

	return afero.Walk(gitFs, "/", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if info.Name() == ".git" {
				return filepath.SkipDir
			}
			return nil
		}
		src, err := gitFs.Open(path)
		if err != nil {
			return err
		}
		return fs.Copy(&fileInfo{
			name: path,
			size: info.Size(),
			mode: info.Mode(),
		}, src)
	})
}

func (g *gitFetcher) fetchWithGoGit(gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)

	storage, err := gitStorage.NewStorage(gitOsFS.New(gitDir))
	if err != nil {
		return err
	}

	errch := make(chan error)
	repch := make(chan *git.Repository)

	srcStr := src.String()
	g.log.Infof("[git] Clone with go-git %s %s in %s", srcStr, branch, gitDir)
	go func() {
		repc, errc := git.Clone(storage, nil, &git.CloneOptions{
			URL:           srcStr,
			Depth:         1,
			SingleBranch:  true,
			ReferenceName: gitPlumbing.ReferenceName(branch),
		})
		if errc != nil {
			errch <- errc
		} else {
			repch <- repc
		}
	}()

	var rep *git.Repository
	select {
	case rep = <-repch:
	case err = <-errch:
		g.log.Errorf("[git] Clone error of %s: %s", srcStr, err.Error())
		return err
	case <-time.After(cloneTimeout):
		g.log.Errorf("[git] Clone timeout of %s", srcStr)
		return errCloneTimeout
	}

	ref, err := rep.Head()
	if err != nil {
		return err
	}

	slug := man.Slug()
	version := man.Version() + "-" + ref.Hash().String()

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil {
		return err
	}
	defer func() {
		if errc := fs.Close(); errc != nil {
			err = errc
		}
	}()
	if exists {
		return nil
	}

	commit, err := rep.CommitObject(ref.Hash())
	if err != nil {
		return err
	}

	files, err := commit.Files()
	if err != nil {
		return err
	}

	return files.ForEach(func(f *gitObject.File) error {
		var r io.ReadCloser
		r, err = f.Reader()
		if err != nil {
			return err
		}
		defer r.Close()
		return fs.Copy(&fileInfo{
			name: f.Name,
			size: f.Size,
			mode: os.FileMode(f.Mode),
		}, r)
	})
}

func getWebBranch(src *url.URL) string {
	if src.Fragment != "" {
		return src.Fragment
	}
	return "HEAD"
}

func getRemoteURL(src *url.URL) (*url.URL, string) {
	branch := src.Fragment
	if branch == "" {
		branch = "master"
	}
	clonedSrc := *src
	clonedSrc.Fragment = ""
	return &clonedSrc, branch
}

func resolveGithubURL(src *url.URL, filename string) (string, error) {
	match := ghURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(ghRawManifestURL, user, project, branch, filename)
	return u, nil
}

func resolveGitlabURL(src *url.URL, filename string) (string, error) {
	match := glURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(glRawManifestURL, src.Host, user, project, branch, filename)
	return u, nil
}

func resolveManifestURL(src *url.URL, filename string) (string, error) {
	// TODO check that it works with a branch
	srccopy, _ := url.Parse(src.String())
	srccopy.Scheme = "http"
	if srccopy.Path == "" || srccopy.Path[len(srccopy.Path)-1] != '/' {
		srccopy.Path += "/"
	}
	srccopy.Path = srccopy.Path + filename
	return srccopy.String(), nil
}

func isGitSSHScheme(scheme string) bool {
	return scheme == "git+ssh" || scheme == "ssh+git"
}

var (
	_ Fetcher = &gitFetcher{}
)
<|endoftext|>


<commit_before>/*
Fully documented and friendly VMWare vSphere API for Go.

Versions supported

vSphere 5.5 and previous versions.

Prerequisite

Before attempting to use this API please take some time to familiarize yourself
with [VMware VI object model](http://www.doublecloud.org/2010/02/object-model-of-vmware-vsphere-api-a-big-picture-in-2-minutes/)

CLI installation

 $ go install

API code generation

 $ govsphere generate

API definitions generation

The generation process is going to create a file called api.json, relative to the path
from where the command is executed.

 $ govsphere scrape

API Documentation

http://godoc.org/github.com/cloudescape/govsphere

License

Copyright 2014 Cloudescape

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.

*/
package vim
<commit_msg>Removes old license<commit_after>/*
Fully documented and friendly VMWare vSphere API for Go.

Versions supported

vSphere 5.5 and previous versions.

Prerequisite

Before attempting to use this API please take some time to familiarize yourself
with [VMware VI object model](http://www.doublecloud.org/2010/02/object-model-of-vmware-vsphere-api-a-big-picture-in-2-minutes/)

CLI installation

 $ go install

API code generation

 $ govsphere generate

API definitions generation

The generation process is going to create a file called api.json, relative to the path
from where the command is executed.

 $ govsphere scrape

API Documentation

http://godoc.org/github.com/cloudescape/govsphere
*/
package vim
<|endoftext|>


<commit_before>// 30 january 2015
package main

import (
	"github.com/andlabs/irksome/iface"
)

// #include "irksome.h"
import "C"

type channel struct {
	name	string
	server	iface.Server
	channel	iface.Channel
}

var chans = make([]*channel, 0, 50)

var addChannel = make(chan *channel)
var channelAdded = make(chan struct{})

var parentServers = make(map[iface.Server]C.gint64)

type sendMessageParams struct {
	message		string
	id			C.gint64
}

var sendMessage = make(chan *sendMessageParams)

func doChannels() {
	for {
		select {
		case cc := <-addChannel:
			chans = append(chans, cc)
			n := C.gint64(len(chans) - 1)
			parent := C.gint64(-1)
			if chans[n].channel != nil {
				parent = parentServers[chans[n].server]
			} else {
				parentServers[chans[n].server] = n
			}
			C.tellUI(C.mAddChannel, strToArg(cc.name), C.TRUE, n, parent)
			<-channelAdded
			// TODO start monitoring
		case m := <-sendMessage:
			// TODO
			_ = m
		}
	}
}
<commit_msg>More integration work. I think I'm going to need to redefine iface again...<commit_after>// 30 january 2015
package main

import (
	"github.com/andlabs/irksome/iface"
)

// #include "irksome.h"
import "C"

type channel struct {
	name	string
	server	iface.Server
	channel	iface.Channel
}

var chans = make([]*channel, 0, 50)

var addChannel = make(chan *channel)
var channelAdded = make(chan struct{})

// TODO merge
var parentServers = make(map[iface.Server]C.gint64)
var channelIDs = make(map[iface.Channel]C.gint64)

type sendMessageParams struct {
	message		string
	id			C.gint64
}

var sendMessage = make(chan *sendMessageParams)
var recvMessage = make(chan iface.Message)

func doChannels() {
	for {
		select {
		case cc := <-addChannel:
			chans = append(chans, cc)
			n := C.gint64(len(chans) - 1)
			parent := C.gint64(-1)
			if chans[n].channel != nil {
				parent = parentServers[chans[n].server]
				channelIDs[chans[n].channel] = n
			} else {
				parentServers[chans[n].server] = n
			}
			C.tellUI(C.mAddChannel, strToArg(cc.name), C.TRUE, n, parent)
			<-channelAdded
		case m := <-sendMessage:
			// TODO
			_ = m
		case m := <-recvMessage:
			// TODO current channels per server
			n, ok := channelIDs[m.Channel()]
			if !ok {
				n = parentServers[m.Server()]
			}
			// TODO
			_ = n
		}
	}
}

func listen(server <|endoftext|>


<commit_before>// Copyright 2015 henrylee2cn Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package surfer

import (
	"compress/flate"
	"compress/gzip"
	"compress/zlib"
	"crypto/tls"
	"io"
	"math/rand"
	"net"
	"net/http"
	"net/http/cookiejar"
	"strings"
	"time"

	"github.com/henrylee2cn/goutil"
	"github.com/henrylee2cn/pholcus/app/downloader/surfer/agent"
)

// Surf is the default Download implementation.
type Surf struct {
	CookieJar *cookiejar.Jar
}

// New 创建一个Surf下载器
func New(jar ...*cookiejar.Jar) Surfer {
	s := new(Surf)
	if len(jar) != 0 {
		s.CookieJar = jar[0]
	} else {
		s.CookieJar, _ = cookiejar.New(nil)
	}
	return s
}

// Download 实现surfer下载器接口
func (self *Surf) Download(req Request) (resp *http.Response, err error) {
	param, err := NewParam(req)
	if err != nil {
		return nil, err
	}
	param.header.Set("Connection", "close")
	param.client = self.buildClient(param)
	resp, err = self.httpRequest(param)

	if err == nil {
		switch resp.Header.Get("Content-Encoding") {
		case "gzip":
			var gzipReader *gzip.Reader
			gzipReader, err = gzip.NewReader(resp.Body)
			if err == nil {
				resp.Body = gzipReader
			}

		case "deflate":
			resp.Body = flate.NewReader(resp.Body)

		case "zlib":
			var readCloser io.ReadCloser
			readCloser, err = zlib.NewReader(resp.Body)
			if err == nil {
				resp.Body = readCloser
			}
		}
	}

	resp = param.writeback(resp)

	return
}

var dnsCache = &DnsCache{ipPortLib: goutil.AtomicMap()}

// DnsCache DNS cache
type DnsCache struct {
	ipPortLib goutil.Map
}

// Reg registers DNS to cache.
func (d *DnsCache) Reg(addr, ipPort string) {
	d.ipPortLib.Store(addr, ipPort)
}

// Query queries DNS from cache.
func (d *DnsCache) Query(addr string) (string, bool) {
	ipPort, ok := d.ipPortLib.Load(addr)
	if !ok {
		return "", false
	}
	return ipPort.(string), true
}

// buildClient creates, configures, and returns a *http.Client type.
func (self *Surf) buildClient(param *Param) *http.Client {
	client := &http.Client{
		CheckRedirect: param.checkRedirect,
	}

	if param.enableCookie {
		client.Jar = self.CookieJar
	}

	transport := &http.Transport{
		Dial: func(network, addr string) (net.Conn, error) {
			var (
				c          net.Conn
				err        error
				ipPort, ok = dnsCache.Query(addr)
			)
			if !ok {
				ipPort = addr
				defer func() {
					if err == nil {
						dnsCache.Reg(addr, c.RemoteAddr().String())
					}
				}()
			}
			c, err = net.DialTimeout(network, ipPort, param.dialTimeout)
			if err != nil {
				return nil, err
			}
			if param.connTimeout > 0 {
				c.SetDeadline(time.Now().Add(param.connTimeout))
			}
			return c, nil
		},
	}

	if param.proxy != nil {
		transport.Proxy = http.ProxyURL(param.proxy)
	}

	if strings.ToLower(param.url.Scheme) == "https" {
		transport.TLSClientConfig = &tls.Config{RootCAs: nil, InsecureSkipVerify: true}
		transport.DisableCompression = true
	}
	client.Transport = transport
	return client
}

// send uses the given *http.Request to make an HTTP request.
func (self *Surf) httpRequest(param *Param) (resp *http.Response, err error) {
	req, err := http.NewRequest(param.method, param.url.String(), param.body)
	if err != nil {
		return nil, err
	}

	req.Header = param.header

	if param.tryTimes <= 0 {
		for {
			resp, err = param.client.Do(req)
			if err != nil {
				if !param.enableCookie {
					l := len(agent.UserAgents["common"])
					r := rand.New(rand.NewSource(time.Now().UnixNano()))
					req.Header.Set("User-Agent", agent.UserAgents["common"][r.Intn(l)])
				}
				time.Sleep(param.retryPause)
				continue
			}
			break
		}
	} else {
		for i := 0; i < param.tryTimes; i++ {
			resp, err = param.client.Do(req)
			if err != nil {
				if !param.enableCookie {
					l := len(agent.UserAgents["common"])
					r := rand.New(rand.NewSource(time.Now().UnixNano()))
					req.Header.Set("User-Agent", agent.UserAgents["common"][r.Intn(l)])
				}
				time.Sleep(param.retryPause)
				continue
			}
			break
		}
	}

	return resp, err
}
<commit_msg>surfer: Optimize DNS cache<commit_after>// Copyright 2015 henrylee2cn Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package surfer

import (
	"compress/flate"
	"compress/gzip"
	"compress/zlib"
	"crypto/tls"
	"io"
	"math/rand"
	"net"
	"net/http"
	"net/http/cookiejar"
	"strings"
	"time"

	"github.com/henrylee2cn/goutil"
	"github.com/henrylee2cn/pholcus/app/downloader/surfer/agent"
)

// Surf is the default Download implementation.
type Surf struct {
	CookieJar *cookiejar.Jar
}

// New 创建一个Surf下载器
func New(jar ...*cookiejar.Jar) Surfer {
	s := new(Surf)
	if len(jar) != 0 {
		s.CookieJar = jar[0]
	} else {
		s.CookieJar, _ = cookiejar.New(nil)
	}
	return s
}

// Download 实现surfer下载器接口
func (self *Surf) Download(req Request) (resp *http.Response, err error) {
	param, err := NewParam(req)
	if err != nil {
		return nil, err
	}
	param.header.Set("Connection", "close")
	param.client = self.buildClient(param)
	resp, err = self.httpRequest(param)

	if err == nil {
		switch resp.Header.Get("Content-Encoding") {
		case "gzip":
			var gzipReader *gzip.Reader
			gzipReader, err = gzip.NewReader(resp.Body)
			if err == nil {
				resp.Body = gzipReader
			}

		case "deflate":
			resp.Body = flate.NewReader(resp.Body)

		case "zlib":
			var readCloser io.ReadCloser
			readCloser, err = zlib.NewReader(resp.Body)
			if err == nil {
				resp.Body = readCloser
			}
		}
	}

	resp = param.writeback(resp)

	return
}

var dnsCache = &DnsCache{ipPortLib: goutil.AtomicMap()}

// DnsCache DNS cache
type DnsCache struct {
	ipPortLib goutil.Map
}

// Reg registers ipPort to DNS cache.
func (d *DnsCache) Reg(addr, ipPort string) {
	d.ipPortLib.Store(addr, ipPort)
}

// Del deletes ipPort from DNS cache.
func (d *DnsCache) Del(addr string) {
	d.ipPortLib.Delete(addr)
}

// Query queries ipPort from DNS cache.
func (d *DnsCache) Query(addr string) (string, bool) {
	ipPort, ok := d.ipPortLib.Load(addr)
	if !ok {
		return "", false
	}
	return ipPort.(string), true
}

// buildClient creates, configures, and returns a *http.Client type.
func (self *Surf) buildClient(param *Param) *http.Client {
	client := &http.Client{
		CheckRedirect: param.checkRedirect,
	}

	if param.enableCookie {
		client.Jar = self.CookieJar
	}

	transport := &http.Transport{
		Dial: func(network, addr string) (net.Conn, error) {
			var (
				c          net.Conn
				err        error
				ipPort, ok = dnsCache.Query(addr)
			)
			if !ok {
				ipPort = addr
				defer func() {
					if err == nil {
						dnsCache.Reg(addr, c.RemoteAddr().String())
					}
				}()
			} else {
				defer func() {
					if err != nil {
						dnsCache.Del(addr)
					}
				}()
			}
			c, err = net.DialTimeout(network, ipPort, param.dialTimeout)
			if err != nil {
				return nil, err
			}
			if param.connTimeout > 0 {
				c.SetDeadline(time.Now().Add(param.connTimeout))
			}
			return c, nil
		},
	}

	if param.proxy != nil {
		transport.Proxy = http.ProxyURL(param.proxy)
	}

	if strings.ToLower(param.url.Scheme) == "https" {
		transport.TLSClientConfig = &tls.Config{RootCAs: nil, InsecureSkipVerify: true}
		transport.DisableCompression = true
	}
	client.Transport = transport
	return client
}

// send uses the given *http.Request to make an HTTP request.
func (self *Surf) httpRequest(param *Param) (resp *http.Response, err error) {
	req, err := http.NewRequest(param.method, param.url.String(), param.body)
	if err != nil {
		return nil, err
	}

	req.Header = param.header

	if param.tryTimes <= 0 {
		for {
			resp, err = param.client.Do(req)
			if err != nil {
				if !param.enableCookie {
					l := len(agent.UserAgents["common"])
					r := rand.New(rand.NewSource(time.Now().UnixNano()))
					req.Header.Set("User-Agent", agent.UserAgents["common"][r.Intn(l)])
				}
				time.Sleep(param.retryPause)
				continue
			}
			break
		}
	} else {
		for i := 0; i < param.tryTimes; i++ {
			resp, err = param.client.Do(req)
			if err != nil {
				if !param.enableCookie {
					l := len(agent.UserAgents["common"])
					r := rand.New(rand.NewSource(time.Now().UnixNano()))
					req.Header.Set("User-Agent", agent.UserAgents["common"][r.Intn(l)])
				}
				time.Sleep(param.retryPause)
				continue
			}
			break
		}
	}

	return resp, err
}
<|endoftext|>


<commit_before>package tensor3

import "runtime"

func init() {
	Hints.Threads = uint(runtime.NumCPU()) - 1
	Hints.DefaultChunkSize = 10000
	//runtime.GOMAXPROCS(1)
}

var Hints struct {
	Threads          uint
	ChunkSizeFixed   bool
	DefaultChunkSize uint
}

// selects parallel application of functions to Vectors and Matrices types (slices of Vector and Matrix types).
// this occurs in chunks whose size is controlled by Hints.
var Parallel bool

// selects parallel application of functions to Matrix components,(its Vector fields).
// only improves performance if using costly functions, non of the built-ins are likely to benefit. YRMV.
var ParallelComponents bool

func chunkSize(l uint) uint {
	if !Hints.ChunkSizeFixed {
		if cs := l / (Hints.Threads + 1); cs > Hints.DefaultChunkSize {
			return cs
		}
	}
	return Hints.DefaultChunkSize
}

// return a channel of Vectors that are chunks of the passed Vectors
func vectorsInChunks(vs Vectors) chan Vectors {
	c := make(chan Vectors, 1)
	cs:=chunkSize(uint(len(vs)))
	lastSplitMax := uint(len(vs))-cs
	go func() {
		var bottom uint
		for top := cs; top < lastSplitMax; top += cs {
			c <- vs[bottom:top]
			bottom = top
		}
		c <- vs[bottom:]
		close(c)
	}()
	return c
}

// return a channel of Matrices that are chunks of the passed Matrices
func matricesInChunks(ms Matrices) chan Matrices {
	c := make(chan Matrices)
	cs:=chunkSize(uint(len(ms)))
	lastSplitMax := uint(len(ms))-cs
	go func() {
		var bottom uint
		for top := cs; top < lastSplitMax; top += cs {
			c <- ms[bottom:top]
			bottom = top
		}
		c <- ms[bottom:]
		close(c)
	}()
	return c
}

// return a channel of VectorRefs that are chunks of the passed VectorRefs
func vectorRefsInChunks(vs VectorRefs) chan VectorRefs {
	c := make(chan VectorRefs, 1)
	cs:=chunkSize(uint(len(vs)))
	lastSplitMax := uint(len(vs))-cs
	go func() {
		var bottom uint
		for top := cs; top < lastSplitMax; top += cs {
			c <- vs[bottom:top]
			bottom = top
		}
		c <- vs[bottom:]
		close(c)
	}()
	return c
}

// return a channel of VectorRefs that are chunks of the passed VectorRefs.
// as an optimisation, which some functions might benefit from, the VectorRefs are reordered so that each chunk contains all/only the values within a region. nearby points are MUCH more likely to be in the same chunk.
func vectorRefsInRegionalChunks(vs VectorRefs) chan VectorRefs {
	c := make(chan VectorRefs, 1)
	cs:=chunkSize(uint(len(vs)))
	if cs>uint(len(vs)){
		c <- vs
		close(c)
		return c
	}
	go func() {
		// sample 5% of points to make guess at distribution
		// TODO sample cubed root number of points?
		sp:=make(VectorRefs,len(vs)/20)
		for i:=range(sp){
			sp[i]=vs[i*20]
		}
		// TODO improve this fixed 8-way only scheme. but if less than 8 cores, is there much point?
		// TODO regions to give chunks of a size similar to other chunking functions.
		average:=sp.Sum()
		divisor:=BaseType(len(sp))
		average.x/=divisor
		average.y/=divisor
		average.z/=divisor
		//(&average).Divide(BaseType(len(sp)))
		var chunks [2][2][2]VectorRefs
		for _,v := range(vs){
			if v.x>average.x {
				if v.y>average.y {
					if v.z>average.z {
						chunks[1][1][1]=append(chunks[1][1][1],v)
					}else{
						chunks[1][1][0]=append(chunks[1][1][0],v)
					}
				}else{
					if v.z>average.z {
						chunks[1][0][1]=append(chunks[1][0][1],v)
					}else{
						chunks[1][0][0]=append(chunks[1][0][0],v)
					}
				}
			}else{
				if v.y>average.y {
					if v.z>average.z {
						chunks[0][1][1]=append(chunks[0][1][1],v)
					}else{
						chunks[0][1][0]=append(chunks[0][1][0],v)
					}
				}else{
					if v.z>average.z {
						chunks[0][0][1]=append(chunks[0][0][1],v)
					}else{
						chunks[0][0][0]=append(chunks[0][0][0],v)
					}
				}
			}
		}
		c <- chunks[0][0][0]
		c <- chunks[0][0][1]
		c <- chunks[0][1][0]
		c <- chunks[0][1][1]
		c <- chunks[1][0][0]
		c <- chunks[1][0][1]
		c <- chunks[1][1][0]
		c <- chunks[1][1][1]
		close(c)
	}()
	return c
}

<commit_msg>add chunking for vectors slices<commit_after>package tensor3

import "runtime"

func init() {
	Hints.Threads = uint(runtime.NumCPU()) - 1
	Hints.DefaultChunkSize = 10000
	//runtime.GOMAXPROCS(1)
}

var Hints struct {
	Threads          uint
	ChunkSizeFixed   bool
	DefaultChunkSize uint
}

// selects parallel application of functions to Vectors and Matrices types (slices of Vector and Matrix types).
// this occurs in chunks whose size is controlled by Hints.
var Parallel bool

// selects parallel application of functions to Matrix components,(its Vector fields).
// only improves performance if using costly functions, non of the built-ins are likely to benefit. YRMV.
var ParallelComponents bool

func chunkSize(l uint) uint {
	if !Hints.ChunkSizeFixed {
		if cs := l / (Hints.Threads + 1); cs > Hints.DefaultChunkSize {
			return cs
		}
	}
	return Hints.DefaultChunkSize
}

// return a channel of Vectors that are chunks of the passed Vectors
func vectorsInChunks(vs Vectors) chan Vectors {
	c := make(chan Vectors, 1)
	cs:=chunkSize(uint(len(vs)))
	lastSplitMax := uint(len(vs))-cs
	go func() {
		var bottom uint
		for top := cs; top < lastSplitMax; top += cs {
			c <- vs[bottom:top]
			bottom = top
		}
		c <- vs[bottom:]
		close(c)
	}()
	return c
}

// return a channel of Matrices that are chunks of the passed Matrices
func matricesInChunks(ms Matrices) chan Matrices {
	c := make(chan Matrices)
	cs:=chunkSize(uint(len(ms)))
	lastSplitMax := uint(len(ms))-cs
	go func() {
		var bottom uint
		for top := cs; top < lastSplitMax; top += cs {
			c <- ms[bottom:top]
			bottom = top
		}
		c <- ms[bottom:]
		close(c)
	}()
	return c
}

// return a channel of VectorRefs that are chunks of the passed VectorRefs
func vectorRefsInChunks(vs VectorRefs) chan VectorRefs {
	c := make(chan VectorRefs, 1)
	cs:=chunkSize(uint(len(vs)))
	lastSplitMax := uint(len(vs))-cs
	go func() {
		var bottom uint
		for top := cs; top < lastSplitMax; top += cs {
			c <- vs[bottom:top]
			bottom = top
		}
		c <- vs[bottom:]
		close(c)
	}()
	return c
}

// return a channel of chunks of, fixed length overlapping slices of, the passed Vectors
// include slices that wrap around, from the end to the start of the Vectors.
// (notice that all the slices are the same provided length.)
// (notice the same Vector, at the ends of the chunks, will in general be in slices in different chunks.)
func vectorSlicesInChunks(vs Vectors,length int,wrap bool) chan []Vectors {
	c := make(chan []Vectors, 1)
	for vsc:=range vectorsInChunks(vs){
		vssc := make([]Vectors,len(vsc))
		for i := range vssc {
			vssc[i]=vsc[i:i+length]
		}
		c <- vssc
	}
	return c
}

// return a channel of VectorRefs that are chunks of the passed VectorRefs.
// as an optimisation, which some functions might benefit from, the VectorRefs are reordered so that each chunk contains all/only the values within a spacial region, so nearby points are MUCH more likely to be in the same chunk.
// keep record of returned chunks to be able to efficiently repeat use a functin on the same vectors.
func vectorRefsInRegionalChunks(vs VectorRefs) chan VectorRefs {
	c := make(chan VectorRefs, 1)
	cs:=chunkSize(uint(len(vs)))
	if cs>uint(len(vs)){
		c <- vs
		close(c)
		return c
	}
	go func() {
		// sample 5% of points to make guess at distribution
		// TODO sample cubed root number of points?
		sp:=make(VectorRefs,len(vs)/20)
		for i:=range(sp){
			sp[i]=vs[i*20]
		}
		// TODO improve this fixed 8-way only scheme. but if less than 8 cores, is there much point?
		// TODO regions to give chunks of a size similar to other chunking functions.
		average:=sp.Sum()
		divisor:=BaseType(len(sp))
		average.x/=divisor
		average.y/=divisor
		average.z/=divisor
		//(&average).Divide(BaseType(len(sp)))
		var chunks [2][2][2]VectorRefs
		for _,v := range(vs){
			if v.x>average.x {
				if v.y>average.y {
					if v.z>average.z {
						chunks[1][1][1]=append(chunks[1][1][1],v)
					}else{
						chunks[1][1][0]=append(chunks[1][1][0],v)
					}
				}else{
					if v.z>average.z {
						chunks[1][0][1]=append(chunks[1][0][1],v)
					}else{
						chunks[1][0][0]=append(chunks[1][0][0],v)
					}
				}
			}else{
				if v.y>average.y {
					if v.z>average.z {
						chunks[0][1][1]=append(chunks[0][1][1],v)
					}else{
						chunks[0][1][0]=append(chunks[0][1][0],v)
					}
				}else{
					if v.z>average.z {
						chunks[0][0][1]=append(chunks[0][0][1],v)
					}else{
						chunks[0][0][0]=append(chunks[0][0][0],v)
					}
				}
			}
		}
		c <- chunks[0][0][0]
		c <- chunks[0][0][1]
		c <- chunks[0][1][0]
		c <- chunks[0][1][1]
		c <- chunks[1][0][0]
		c <- chunks[1][0][1]
		c <- chunks[1][1][0]
		c <- chunks[1][1][1]
		close(c)
	}()
	return c
}

<|endoftext|>