<commit_before>package main

import (
	"context"
	"encoding/json"
	"html/template"
	"net/http"

	"github.com/elazarl/go-bindata-assetfs"
	"github.com/gorilla/websocket"
	bloomsky "github.com/patrickalin/bloomsky-api-go"
	"github.com/patrickalin/bloomsky-client-go/assembly-assetfs"
	"github.com/patrickalin/bloomsky-client-go/utils"
	"github.com/sirupsen/logrus"
)

var (
	conn       *websocket.Conn
	mybloomsky bloomsky.BloomskyStructure
	msgJSON    []byte
)

type httpServer struct {
	bloomskyMessageToHTTP chan bloomsky.BloomskyStructure
	httpServ              *http.Server
}

func (httpServ *httpServer) listen(context context.Context) {
	go func() {
		for {
			var err error
			mybloomsky := <-httpServ.bloomskyMessageToHTTP
			msgJSON, err = json.Marshal(mybloomsky)
			log.Debugf("JSON : %s", msgJSON)

			if err != nil {
				log.Infof("Marshal json Error: %v", err)
				return
			}

			if conn != nil {
				err = conn.WriteMessage(websocket.TextMessage, msgJSON)
				if err != nil {
					log.Infof("Impossible to write to websocket : %v", err)
				}
			}
			log.Debug("Message send to browser")
		}
	}()
}

// Websocket handler to send data
func (httpServ *httpServer) refreshdata(w http.ResponseWriter, r *http.Request) {
	log.Debugf("Refresdata WS handle Send JSON : %s", msgJSON)

	upgrader := websocket.Upgrader{}
	var err error

	conn, err = upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Errorf("Upgrade upgrader : %v", err)
		return
	}

	if err = conn.WriteMessage(websocket.TextMessage, msgJSON); err != nil {
		log.Errorf("Impossible to write to websocket : %v", err)
	}

}

//Handler for the page without data
func (httpServ *httpServer) home(w http.ResponseWriter, r *http.Request) {
	log.Debugf("Home Http handle Send JSON : %s", msgJSON)

	var err error
	var templateHeader *template.Template
	var templateBody *template.Template

	templateHeader = utils.GetHtmlTemplate("bloomsky_header.html", "tmpl/bloomsky_header.html", map[string]interface{}{"T": config.translateFunc}, config.dev)

	err = templateHeader.Execute(w, "ws://"+r.Host+"/refreshdata")
	if err != nil {
		log.Fatalf("Write part 1 : %v", err)
	}
	templateBody = utils.GetHtmlTemplate("bloomsky_body.html", "tmpl/bloomsky_body.html", map[string]interface{}{"T": config.translateFunc}, config.dev)

	err = templateBody.Execute(w, mybloomsky)
	if err != nil {
		log.Fatalf("Write part 2 : %v", err)
	}
}

//createWebServer create web server
func createWebServer(in chan bloomsky.BloomskyStructure, HTTPPort string) (*httpServer, error) {
	server := &httpServer{bloomskyMessageToHTTP: in}

	fs := http.FileServer(&assetfs.AssetFS{Asset: assemblyAssetfs.Asset, AssetDir: assemblyAssetfs.AssetDir, AssetInfo: assemblyAssetfs.AssetInfo, Prefix: "static"})

	s := http.NewServeMux()

	s.Handle("/static/", http.StripPrefix("/static/", fs))
	s.HandleFunc("/refreshdata", server.refreshdata)
	s.HandleFunc("/", server.home)

	h := &http.Server{Addr: HTTPPort, Handler: s}
	go func() {
		if err := h.ListenAndServe(); err != nil {
			log.Errorf("Error when I create the server : %v", err)
		}
	}()
	logrus.Infof("Server listen on port %s", HTTPPort)
	server.httpServ = h
	return server, nil
}
<commit_msg>Simplify err<commit_after>package main

import (
	"context"
	"encoding/json"
	"html/template"
	"net/http"

	"github.com/elazarl/go-bindata-assetfs"
	"github.com/gorilla/websocket"
	bloomsky "github.com/patrickalin/bloomsky-api-go"
	"github.com/patrickalin/bloomsky-client-go/assembly-assetfs"
	"github.com/patrickalin/bloomsky-client-go/utils"
	"github.com/sirupsen/logrus"
)

var (
	conn       *websocket.Conn
	mybloomsky bloomsky.BloomskyStructure
	msgJSON    []byte
)

type httpServer struct {
	bloomskyMessageToHTTP chan bloomsky.BloomskyStructure
	httpServ              *http.Server
}

func (httpServ *httpServer) listen(context context.Context) {
	go func() {
		for {
			var err error
			mybloomsky := <-httpServ.bloomskyMessageToHTTP
			msgJSON, err = json.Marshal(mybloomsky)
			log.Debugf("JSON : %s", msgJSON)

			if err != nil {
				log.Infof("Marshal json Error: %v", err)
				return
			}

			if conn != nil {
				err = conn.WriteMessage(websocket.TextMessage, msgJSON)
				if err != nil {
					log.Infof("Impossible to write to websocket : %v", err)
				}
			}
			log.Debug("Message send to browser")
		}
	}()
}

// Websocket handler to send data
func (httpServ *httpServer) refreshdata(w http.ResponseWriter, r *http.Request) {
	log.Debugf("Refresdata WS handle Send JSON : %s", msgJSON)

	upgrader := websocket.Upgrader{}
	var err error

	conn, err = upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Errorf("Upgrade upgrader : %v", err)
		return
	}

	if err = conn.WriteMessage(websocket.TextMessage, msgJSON); err != nil {
		log.Errorf("Impossible to write to websocket : %v", err)
	}

}

//Handler for the page without data
func (httpServ *httpServer) home(w http.ResponseWriter, r *http.Request) {
	log.Debugf("Home Http handle Send JSON : %s", msgJSON)

	var templateHeader *template.Template
	var templateBody *template.Template

	templateHeader = utils.GetHtmlTemplate("bloomsky_header.html", "tmpl/bloomsky_header.html", map[string]interface{}{"T": config.translateFunc}, config.dev)

	if err := templateHeader.Execute(w, "ws://"+r.Host+"/refreshdata"); err != nil {
		log.Fatalf("Write part 1 : %v", err)
	}
	templateBody = utils.GetHtmlTemplate("bloomsky_body.html", "tmpl/bloomsky_body.html", map[string]interface{}{"T": config.translateFunc}, config.dev)

	if err := templateBody.Execute(w, mybloomsky); err != nil {
		log.Fatalf("Write part 2 : %v", err)
	}
}

//createWebServer create web server
func createWebServer(in chan bloomsky.BloomskyStructure, HTTPPort string) (*httpServer, error) {
	server := &httpServer{bloomskyMessageToHTTP: in}

	fs := http.FileServer(&assetfs.AssetFS{Asset: assemblyAssetfs.Asset, AssetDir: assemblyAssetfs.AssetDir, AssetInfo: assemblyAssetfs.AssetInfo, Prefix: "static"})

	s := http.NewServeMux()

	s.Handle("/static/", http.StripPrefix("/static/", fs))
	s.HandleFunc("/refreshdata", server.refreshdata)
	s.HandleFunc("/", server.home)

	h := &http.Server{Addr: HTTPPort, Handler: s}
	go func() {
		if err := h.ListenAndServe(); err != nil {
			log.Errorf("Error when I create the server : %v", err)
		}
	}()
	logrus.Infof("Server listen on port %s", HTTPPort)
	server.httpServ = h
	return server, nil
}
<|endoftext|>


<commit_before>// Copyright ©2016 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fd_test

import (
	"fmt"
	"math"

	"github.com/gonum/diff/fd"
)

func ExampleDerivative() {
	f := func(x float64) float64 {
		return math.Sin(x)
	}
	// Compute the first derivative of f at 0 using the default settings.
	fmt.Println("f'(0) ≈", fd.Derivative(f, 0, nil))
	// Compute the first derivative of f at 0 using the forward approximation
	// with a custom step size.
	df := fd.Derivative(f, 0, &fd.Settings{
		Formula: fd.Forward,
		Step:    1e-8,
	})
	fmt.Println("f'(0) ≈", df)

	f = func(x float64) float64 {
		return math.Pow(math.Cos(x), 3)
	}
	// Compute the second derivative of f at 0 using the centered
	// approximation, a custom step size, concurrent evaluation, and a known
	// function value at x.
	df = fd.Derivative(f, 0, &fd.Settings{
		Formula:     fd.Central2nd,
		Step:        1e-4,
		Concurrent:  true,
		OriginKnown: true,
		OriginValue: f(0),
	})
	fmt.Println("f''(0) ≈", df)

	// Output:
	// f'(0) ≈ 0.999999999994
	// f'(0) ≈ 1
	// f''(0) ≈ -2.999999981767587
}
<commit_msg>fd: update example for Derivative<commit_after>// Copyright ©2016 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fd_test

import (
	"fmt"
	"math"

	"github.com/gonum/diff/fd"
)

func ExampleDerivative() {
	f := func(x float64) float64 {
		return math.Sin(x)
	}
	// Compute the first derivative of f at 0 using the default settings.
	fmt.Println("f'(0) ≈", fd.Derivative(f, 0, nil))
	// Compute the first derivative of f at 0 using the forward approximation
	// with a custom step size.
	df := fd.Derivative(f, 0, &fd.Settings{
		Formula: fd.Forward,
		Step:    1e-3,
	})
	fmt.Println("f'(0) ≈", df)

	f = func(x float64) float64 {
		return math.Pow(math.Cos(x), 3)
	}
	// Compute the second derivative of f at 0 using
	// the centered approximation, concurrent evaluation,
	// and a known function value at x.
	df = fd.Derivative(f, 0, &fd.Settings{
		Formula:     fd.Central2nd,
		Concurrent:  true,
		OriginKnown: true,
		OriginValue: f(0),
	})
	fmt.Println("f''(0) ≈", df)

	// Output:
	// f'(0) ≈ 1
	// f'(0) ≈ 0.9999998333333416
	// f''(0) ≈ -2.999999981767587
}
<|endoftext|>


<commit_before>package testing

import (
	"net/http"
	"net/url"
	"time"
)

type TestHTTPServer struct {
	URL      string
	started  bool
	request  chan *http.Request
	response chan *testResponse
}

type testResponse struct {
	Status  int
	Headers map[string]string
	Body    string
}

func NewTestHTTPServer(url string) *TestHTTPServer {
	return &TestHTTPServer{URL: url}
}

func (s *TestHTTPServer) Start() {
	if s.started {
		return
	}
	s.started = true
	s.request = make(chan *http.Request, 64)
	s.response = make(chan *testResponse, 64)
	url, _ := url.Parse(s.URL)
	go http.ListenAndServe(url.Host, s)
	s.PrepareResponse(202, nil, "Nothing.")
	for {
		resp, err := http.Get(s.URL)
		if err == nil && resp.StatusCode == 202 {
			break
		}
		time.Sleep(1e8)
	}
	s.WaitRequest()
}

func (s *TestHTTPServer) FlushRequests() {
	for {
		select {
		case <-s.request:
		default:
			return
		}
	}
}

func (s *TestHTTPServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	s.request <- req
	var resp *testResponse
	resp = <-s.response
	if resp.Status != 0 {
		w.WriteHeader(resp.Status)
	}
	w.Write([]byte(resp.Body))
}

func (s *TestHTTPServer) WaitRequest() *http.Request {
	req := <-s.request
	req.ParseForm()
	return req
}

func (s *TestHTTPServer) PrepareResponse(status int, headers map[string]string, body string) {
	s.response <- &testResponse{status, headers, body}
}
<commit_msg>testing: added timeout to WaitRequest<commit_after>package testing

import (
	"errors"
	"net/http"
	"net/url"
	"time"
)

type TestHTTPServer struct {
	URL      string
	started  bool
	request  chan *http.Request
	response chan *testResponse
}

type testResponse struct {
	Status  int
	Headers map[string]string
	Body    string
}

func NewTestHTTPServer(url string) *TestHTTPServer {
	return &TestHTTPServer{URL: url}
}

func (s *TestHTTPServer) Start() {
	if s.started {
		return
	}
	s.started = true
	s.request = make(chan *http.Request, 64)
	s.response = make(chan *testResponse, 64)
	url, _ := url.Parse(s.URL)
	go http.ListenAndServe(url.Host, s)
	s.PrepareResponse(202, nil, "Nothing.")
	for {
		resp, err := http.Get(s.URL)
		if err == nil && resp.StatusCode == 202 {
			break
		}
		time.Sleep(1e8)
	}
	s.WaitRequest(1e18)
}

func (s *TestHTTPServer) FlushRequests() {
	for {
		select {
		case <-s.request:
		default:
			return
		}
	}
}

func (s *TestHTTPServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	s.request <- req
	var resp *testResponse
	resp = <-s.response
	if resp.Status != 0 {
		w.WriteHeader(resp.Status)
	}
	w.Write([]byte(resp.Body))
}

func (s *TestHTTPServer) WaitRequest(timeout time.Duration) (*http.Request, error) {
	select {
	case req := <-s.request:
		req.ParseForm()
		return req, nil
	case <-time.After(timeout):
	}
	return nil, errors.New("timed out")
}

func (s *TestHTTPServer) PrepareResponse(status int, headers map[string]string, body string) {
	s.response <- &testResponse{status, headers, body}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package garbage

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"

	"code.google.com/p/go.benchmarks/driver"
)

func init() {
	driver.Register("garbage", benchmark)
}

type ParsedPackage map[string]*ast.Package

var (
	parsed []ParsedPackage
)

func benchmark() driver.Result {
	if parsed == nil {
		mem := packageMemConsumption()
		avail := (driver.BenchMem << 20) * 4 / 5 // 4/5 to account for non-heap memory
		npkg := avail / mem / 2                  // 2 to account for GOGC=100
		parsed = make([]ParsedPackage, npkg)
		for n := 0; n < 2; n++ { // warmup GC
			for i := range parsed {
				parsed[i] = parsePackage()
			}
		}
		fmt.Printf("consumption=%vKB npkg=%d\n", mem>>10, npkg)
	}
	return driver.Benchmark(benchmarkN)
}

func benchmarkN(N uint64) {
	P := runtime.GOMAXPROCS(0)
	// Create G goroutines, but only 2*P of them parse at the same time.
	G := 1024
	gate := make(chan bool, 2*P)
	var mu sync.Mutex
	var wg sync.WaitGroup
	wg.Add(G)
	remain := int64(N)
	pos := 0
	for g := 0; g < G; g++ {
		go func() {
			defer wg.Done()
			for atomic.AddInt64(&remain, -1) >= 0 {
				gate <- true
				p := parsePackage()
				mu.Lock()
				// Overwrite only half of the array,
				// the other part represents "old" generation.
				parsed[pos%(len(parsed)/2)] = p
				pos++
				mu.Unlock()
				<-gate
			}
		}()
	}
	wg.Wait()
}

// packageMemConsumption returns memory consumption of a single parsed package.
func packageMemConsumption() int {
	// One GC does not give precise results,
	// because concurrent sweep may be still in progress.
	runtime.GC()
	runtime.GC()
	ms0 := new(runtime.MemStats)
	runtime.ReadMemStats(ms0)
	const N = 10
	var parsed [N]ParsedPackage
	for i := range parsed {
		parsed[i] = parsePackage()
	}
	runtime.GC()
	runtime.GC()
	// Keep it alive.
	if parsed[0] == nil {
		fmt.Println(&parsed)
	}
	ms1 := new(runtime.MemStats)
	runtime.ReadMemStats(ms1)
	mem := int(ms1.Alloc-ms0.Alloc) / N
	if mem < 1<<16 {
		mem = 1 << 16
	}
	return mem
}

// parsePackage parses and returns net/http package.
func parsePackage() ParsedPackage {
	pkgname := "http"
	dirpath := runtime.GOROOT() + "/src/pkg/net/http"
	// filter function to select the desired .go files
	filter := func(d os.FileInfo) bool {
		if isPkgFile(d) {
			// Some directories contain main packages: Only accept
			// files that belong to the expected package so that
			// parser.ParsePackage doesn't return "multiple packages
			// found" errors.
			// Additionally, accept the special package name
			// fakePkgName if we are looking at cmd documentation.
			name := pkgName(dirpath + "/" + d.Name())
			return name == pkgname
		}
		return false
	}

	// get package AST
	pkgs, err := parser.ParseDir(token.NewFileSet(), dirpath, filter, parser.ParseComments)
	if err != nil {
		println("parse", dirpath, err.Error())
		panic("fail")
	}
	return pkgs
}

func isGoFile(dir os.FileInfo) bool {
	return !dir.IsDir() &&
		!strings.HasPrefix(dir.Name(), ".") && // ignore .files
		path.Ext(dir.Name()) == ".go"
}

func isPkgFile(dir os.FileInfo) bool {
	return isGoFile(dir) &&
		!strings.HasSuffix(dir.Name(), "_test.go") // ignore test files
}

func pkgName(filename string) string {
	file, err := parser.ParseFile(token.NewFileSet(), filename, nil, parser.PackageClauseOnly)
	if err != nil || file == nil {
		return ""
	}
	return file.Name.Name
}
<commit_msg>go.benchmarks: fix garbage benchmark after src/pkg change<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package garbage

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"

	"code.google.com/p/go.benchmarks/driver"
)

func init() {
	driver.Register("garbage", benchmark)
}

type ParsedPackage map[string]*ast.Package

var (
	parsed []ParsedPackage
)

func benchmark() driver.Result {
	if parsed == nil {
		mem := packageMemConsumption()
		avail := (driver.BenchMem << 20) * 4 / 5 // 4/5 to account for non-heap memory
		npkg := avail / mem / 2                  // 2 to account for GOGC=100
		parsed = make([]ParsedPackage, npkg)
		for n := 0; n < 2; n++ { // warmup GC
			for i := range parsed {
				parsed[i] = parsePackage()
			}
		}
		fmt.Printf("consumption=%vKB npkg=%d\n", mem>>10, npkg)
	}
	return driver.Benchmark(benchmarkN)
}

func benchmarkN(N uint64) {
	P := runtime.GOMAXPROCS(0)
	// Create G goroutines, but only 2*P of them parse at the same time.
	G := 1024
	gate := make(chan bool, 2*P)
	var mu sync.Mutex
	var wg sync.WaitGroup
	wg.Add(G)
	remain := int64(N)
	pos := 0
	for g := 0; g < G; g++ {
		go func() {
			defer wg.Done()
			for atomic.AddInt64(&remain, -1) >= 0 {
				gate <- true
				p := parsePackage()
				mu.Lock()
				// Overwrite only half of the array,
				// the other part represents "old" generation.
				parsed[pos%(len(parsed)/2)] = p
				pos++
				mu.Unlock()
				<-gate
			}
		}()
	}
	wg.Wait()
}

// packageMemConsumption returns memory consumption of a single parsed package.
func packageMemConsumption() int {
	// One GC does not give precise results,
	// because concurrent sweep may be still in progress.
	runtime.GC()
	runtime.GC()
	ms0 := new(runtime.MemStats)
	runtime.ReadMemStats(ms0)
	const N = 10
	var parsed [N]ParsedPackage
	for i := range parsed {
		parsed[i] = parsePackage()
	}
	runtime.GC()
	runtime.GC()
	// Keep it alive.
	if parsed[0] == nil {
		fmt.Println(&parsed)
	}
	ms1 := new(runtime.MemStats)
	runtime.ReadMemStats(ms1)
	mem := int(ms1.Alloc-ms0.Alloc) / N
	if mem < 1<<16 {
		mem = 1 << 16
	}
	return mem
}

// parsePackage parses and returns net/http package.
func parsePackage() ParsedPackage {
	pkgname := "http"
	dirpath := runtime.GOROOT() + "/src/net/http"
	// filter function to select the desired .go files
	filter := func(d os.FileInfo) bool {
		if isPkgFile(d) {
			// Some directories contain main packages: Only accept
			// files that belong to the expected package so that
			// parser.ParsePackage doesn't return "multiple packages
			// found" errors.
			// Additionally, accept the special package name
			// fakePkgName if we are looking at cmd documentation.
			name := pkgName(dirpath + "/" + d.Name())
			return name == pkgname
		}
		return false
	}

	// get package AST
	pkgs, err := parser.ParseDir(token.NewFileSet(), dirpath, filter, parser.ParseComments)
	if err != nil {
		println("parse", dirpath, err.Error())
		panic("fail")
	}
	return pkgs
}

func isGoFile(dir os.FileInfo) bool {
	return !dir.IsDir() &&
		!strings.HasPrefix(dir.Name(), ".") && // ignore .files
		path.Ext(dir.Name()) == ".go"
}

func isPkgFile(dir os.FileInfo) bool {
	return isGoFile(dir) &&
		!strings.HasSuffix(dir.Name(), "_test.go") // ignore test files
}

func pkgName(filename string) string {
	file, err := parser.ParseFile(token.NewFileSet(), filename, nil, parser.PackageClauseOnly)
	if err != nil || file == nil {
		return ""
	}
	return file.Name.Name
}
<|endoftext|>


<commit_before>package gen_test

import (
	"testing"

	"github.com/leanovate/gopter"
)

func commonGeneratorTest(t *testing.T, name string, gen gopter.Gen, valueCheck func(interface{}) bool) {
	for i := 0; i < 100; i++ {
		value, ok := gen.Sample()

		if !ok || value == nil {
			t.Errorf("Invalid generator result (%s): %#v", name, value)
		} else if !valueCheck(value) {
			t.Errorf("Invalid value (%s): %#v", name, value)
		}
	}
}
<commit_msg>Expand the commonGeneratorTest so that it exercises and makes assertions about shrinks.<commit_after>package gen_test

import (
	"testing"

	"github.com/leanovate/gopter"
)

func commonGeneratorTest(t *testing.T, name string, gen gopter.Gen, valueCheck func(interface{}) bool) {
	for i := 0; i < 100; i++ {
		value, ok := gen.Sample()

		if !ok || value == nil {
			t.Errorf("Invalid generator result (%s): %#v", name, value)
		} else if !valueCheck(value) {
			t.Errorf("Invalid value (%s): %#v", name, value)
		}

		genResult := gen(gopter.DefaultGenParameters())
		if genResult.Shrinker != nil {
			value, ok := genResult.Retrieve()
			if !ok || value == nil {
				t.Errorf("Invalid generator result (%s): %#v", name, value)
			} else {
				shrink := genResult.Shrinker(value)
				shrunkValue, ok := shrink()
				if ok && !valueCheck(shrunkValue) {
					t.Errorf("Invalid shrunk value (%s): %#v", name, shrunkValue)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"testing"

	"apigov.dev/registry/connection"
	"apigov.dev/registry/rpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

func unavailable(err error) bool {
	if err == nil {
		return false
	}
	st, ok := status.FromError(err)
	if !ok {
		return false
	}
	return st.Code() == codes.Unavailable
}

func check(t *testing.T, message string, err error) {
	if unavailable(err) {
		t.Logf("Unable to connect to registry server. Is it running?")
		t.FailNow()
	}
	if err != nil {
		t.Errorf(message, err.Error())
	}
}

func readAndGZipFile(filename string) (*bytes.Buffer, error) {
	fileBytes, err := ioutil.ReadFile(filename)
	var buf bytes.Buffer
	zw, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression)
	_, err = zw.Write(fileBytes)
	if err != nil {
		return nil, err
	}
	if err := zw.Close(); err != nil {
		return nil, err
	}
	return &buf, nil
}

func hashForBytes(b []byte) string {
	h := sha1.New()
	h.Write(b)
	bs := h.Sum(nil)
	return fmt.Sprintf("%x", bs)
}

func listAllSpecs(ctx context.Context, registryClient connection.Client) []*rpc.Spec {
	specs := make([]*rpc.Spec, 0)
	req := &rpc.ListSpecsRequest{
		Parent: "projects/sample/products/-/versions/-",
	}
	it := registryClient.ListSpecs(ctx, req)
	for {
		spec, err := it.Next()
		if err == nil {
			specs = append(specs, spec)
		} else {
			break
		}
	}
	return specs
}

func listAllSpecRevisionIDs(ctx context.Context, registryClient connection.Client) []string {
	revisionIDs := make([]string, 0)
	req := &rpc.ListSpecRevisionsRequest{
		Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml",
	}
	it := registryClient.ListSpecRevisions(ctx, req)
	for {
		spec, err := it.Next()
		if err == nil {
			revisionIDs = append(revisionIDs, spec.RevisionId)
		} else {
			break
		}
	}
	return revisionIDs
}

func TestDemo(t *testing.T) {
	var revisionIDs []string      // holds revision ids from queries
	var specs []*rpc.Spec         // holds specs from queries
	var originalRevisionID string // revision id of first revision
	var originalHash string       // hash of first revision

	// Create a registry client.
	ctx := context.Background()
	registryClient, err := connection.NewClient(ctx)
	if err != nil {
		t.Logf("Failed to create client: %+v", err)
		t.FailNow()
	}
	defer registryClient.Close()
	// Clear the sample project.
	{
		req := &rpc.DeleteProjectRequest{
			Name: "projects/sample",
		}
		err = registryClient.DeleteProject(ctx, req)
		check(t, "Failed to delete sample project: %+v", err)
	}
	// Create the sample project.
	{
		req := &rpc.CreateProjectRequest{
			ProjectId: "sample",
		}
		project, err := registryClient.CreateProject(ctx, req)
		check(t, "error creating project %s", err)
		if project.GetName() != "projects/sample" {
			t.Errorf("Invalid project name %s", project.GetName())
		}
	}
	// List the sample project. This should return exactly one result.
	{
		req := &rpc.ListProjectsRequest{
			Filter: "project_id == 'sample'",
		}
		count := 0
		it := registryClient.ListProjects(ctx, req)
		for {
			project, err := it.Next()
			if err == nil {
				if project.Name != "projects/sample" {
					t.Errorf("Invalid project name: %s", project.Name)
				}
				count++
			} else {
				break
			}
		}
		if count != 1 {
			t.Errorf("Invalid project count: %d", count)
		}
	}
	// Get the sample project.
	{
		req := &rpc.GetProjectRequest{
			Name: "projects/sample",
		}
		project, err := registryClient.GetProject(ctx, req)
		check(t, "error getting project %s", err)
		if project.Name != "projects/sample" {
			t.Errorf("Invalid project name: %s", project.Name)
		}
	}
	// Create the petstore product.
	{
		req := &rpc.CreateProductRequest{
			Parent:    "projects/sample",
			ProductId: "petstore",
		}
		_, err := registryClient.CreateProduct(ctx, req)
		check(t, "error creating product %s", err)
	}
	// Create the petstore 1.0.0 version.
	{
		req := &rpc.CreateVersionRequest{
			Parent:    "projects/sample/products/petstore",
			VersionId: "1.0.0",
		}
		_, err := registryClient.CreateVersion(ctx, req)
		check(t, "error creating version %s", err)
	}
	// Upload the petstore 1.0.0 OpenAPI spec.
	{
		buf, err := readAndGZipFile("sample/petstore/1.0.0/openapi.yaml@r0")
		check(t, "error reading spec", err)
		req := &rpc.CreateSpecRequest{
			Parent: "projects/sample/products/petstore/versions/1.0.0",
			SpecId: "openapi.yaml",
			Spec: &rpc.Spec{
				Style:    "openapi/v3+gzip",
				Contents: buf.Bytes(),
			},
		}
		_, err = registryClient.CreateSpec(ctx, req)
		check(t, "error creating spec %s", err)
	}
	// Update the OpenAPI spec three times with different revisions.
	for _, filename := range []string{
		"sample/petstore/1.0.0/openapi.yaml@r1",
		"sample/petstore/1.0.0/openapi.yaml@r2",
		"sample/petstore/1.0.0/openapi.yaml@r3",
	} {
		buf, err := readAndGZipFile(filename)
		check(t, "error reading spec", err)
		req := &rpc.UpdateSpecRequest{
			Spec: &rpc.Spec{
				Name:     "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml",
				Contents: buf.Bytes(),
			},
		}
		_, err = registryClient.UpdateSpec(ctx, req)
		check(t, "error updating spec %s", err)
	}
	// List the spec revisions.
	{
		revisionIDs = listAllSpecRevisionIDs(ctx, registryClient)
		if len(revisionIDs) != 4 {
			t.Errorf("Incorrect revision count: %d", len(revisionIDs))
		}
	}
	// Check the hash of the original revision.
	if len(revisionIDs) > 0 {
		originalRevisionID = revisionIDs[len(revisionIDs)-1]
		req := &rpc.GetSpecRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml" + "@" + originalRevisionID,
		}
		spec, err := registryClient.GetSpec(ctx, req)
		check(t, "error getting spec %s", err)
		originalHash = spec.Hash
		// compute the hash of the original file
		buf, err := readAndGZipFile("sample/petstore/1.0.0/openapi.yaml@r0")
		check(t, "error reading spec", err)
		hash2 := hashForBytes(buf.Bytes())
		if originalHash != hash2 {
			t.Errorf("Hash mismatch %s != %s", originalHash, hash2)
		}
	}
	// List specs; there should be only one.
	{
		specs = listAllSpecs(ctx, registryClient)
		if len(specs) != 1 {
			t.Errorf("Incorrect spec count: %d", len(specs))
		}
	}
	// tag a spec revision
	{
		req := &rpc.TagSpecRevisionRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml" + "@" + originalRevisionID,
			Tag:  "og",
		}
		taggedSpec, err := registryClient.TagSpecRevision(ctx, req)
		check(t, "error tagging spec", err)
		if taggedSpec.Name != "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml@og" {
			t.Errorf("Incorrect name of tagged spec: %s", taggedSpec.Name)
		}
		if taggedSpec.Hash != originalHash {
			t.Errorf("Incorrect hash for tagged spec: %s", taggedSpec.Hash)
		}
	}
	// tag the tagged revision
	{
		req := &rpc.TagSpecRevisionRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml@og",
			Tag:  "first",
		}
		taggedSpec, err := registryClient.TagSpecRevision(ctx, req)
		check(t, "error tagging spec", err)
		if taggedSpec.Name != "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml@first" {
			t.Errorf("Incorrect name of tagged spec: %s", taggedSpec.Name)
		}
		if taggedSpec.Hash != originalHash {
			t.Errorf("Incorrect hash for tagged spec: %s", taggedSpec.Hash)
		}
	}
	// get a spec by its tag
	{
		req := &rpc.GetSpecRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml@first",
		}
		spec, err := registryClient.GetSpec(ctx, req)
		check(t, "error getting spec %s", err)
		if spec.Hash != originalHash {
			t.Errorf("Incorrect hash for spec retrieved by tag: %s", spec.Hash)
		}
	}
	// rollback a spec revision (this creates a new revision that's a copy)
	{
		req := &rpc.RollbackSpecRequest{
			Name:       "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml",
			RevisionId: "og",
		}
		spec, err := registryClient.RollbackSpec(ctx, req)
		check(t, "error rolling back spec %s", err)
		if spec.Hash != originalHash {
			t.Errorf("Incorrect hash for rolled-back spec: %s", spec.Hash)
		}
	}
	// List specs; there should be only one.
	{
		specs = listAllSpecs(ctx, registryClient)
		if len(specs) != 1 {
			t.Errorf("Incorrect spec count: %d", len(specs))
		}
	}
	// list spec revisions, there should now be five
	{
		revisionIDs = listAllSpecRevisionIDs(ctx, registryClient)
		if len(revisionIDs) != 5 {
			t.Errorf("Incorrect revision count: %d", len(revisionIDs))
		}
	}
	// delete a spec revision
	{
		req := &rpc.DeleteSpecRevisionRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml@og",
		}
		err := registryClient.DeleteSpecRevision(ctx, req)
		check(t, "error deleting spec revision %s", err)
	}
	// list specs, there should be only one
	{
		specs = listAllSpecs(ctx, registryClient)
		if len(specs) != 1 {
			t.Errorf("Incorrect spec count: %d", len(specs))
		}
	}
	// list spec revisions, there should be four
	{
		revisionIDs = listAllSpecRevisionIDs(ctx, registryClient)
		if len(revisionIDs) != 4 {
			t.Errorf("Incorrect revision count: %d", len(revisionIDs))
		}
	}
	// delete the spec
	{
		req := &rpc.DeleteSpecRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml",
		}
		err := registryClient.DeleteSpec(ctx, req)
		check(t, "error deleting spec %s", err)
	}
	// list spec revisions, there should be none
	{
		revisionIDs = listAllSpecRevisionIDs(ctx, registryClient)
		if len(revisionIDs) != 0 {
			t.Errorf("Incorrect revision count: %d", len(revisionIDs))
		}
	}
}
<commit_msg>minor edits to API tests.<commit_after>// Copyright 2020 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"testing"

	"apigov.dev/registry/connection"
	"apigov.dev/registry/rpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

func unavailable(err error) bool {
	if err == nil {
		return false
	}
	st, ok := status.FromError(err)
	if !ok {
		return false
	}
	return st.Code() == codes.Unavailable
}

func check(t *testing.T, message string, err error) {
	if unavailable(err) {
		t.Logf("Unable to connect to registry server. Is it running?")
		t.FailNow()
	}
	if err != nil {
		t.Errorf(message, err.Error())
	}
}

func readAndGZipFile(filename string) (*bytes.Buffer, error) {
	fileBytes, err := ioutil.ReadFile(filename)
	var buf bytes.Buffer
	zw, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression)
	_, err = zw.Write(fileBytes)
	if err != nil {
		return nil, err
	}
	if err := zw.Close(); err != nil {
		return nil, err
	}
	return &buf, nil
}

func hashForBytes(b []byte) string {
	h := sha1.New()
	h.Write(b)
	bs := h.Sum(nil)
	return fmt.Sprintf("%x", bs)
}

func listAllSpecs(ctx context.Context, registryClient connection.Client) []*rpc.Spec {
	specs := make([]*rpc.Spec, 0)
	req := &rpc.ListSpecsRequest{
		Parent: "projects/sample/products/-/versions/-",
	}
	it := registryClient.ListSpecs(ctx, req)
	for {
		spec, err := it.Next()
		if err == nil {
			specs = append(specs, spec)
		} else {
			break
		}
	}
	return specs
}

func listAllSpecRevisionIDs(ctx context.Context, registryClient connection.Client) []string {
	revisionIDs := make([]string, 0)
	req := &rpc.ListSpecRevisionsRequest{
		Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml",
	}
	it := registryClient.ListSpecRevisions(ctx, req)
	for {
		spec, err := it.Next()
		if err == nil {
			revisionIDs = append(revisionIDs, spec.RevisionId)
		} else {
			break
		}
	}
	return revisionIDs
}

func TestSample(t *testing.T) {
	var revisionIDs []string      // holds revision ids from queries
	var specs []*rpc.Spec         // holds specs from queries
	var originalRevisionID string // revision id of first revision
	var originalHash string       // hash of first revision

	// Create a registry client.
	ctx := context.Background()
	registryClient, err := connection.NewClient(ctx)
	if err != nil {
		t.Logf("Failed to create client: %+v", err)
		t.FailNow()
	}
	defer registryClient.Close()
	// Clear the sample project.
	{
		req := &rpc.DeleteProjectRequest{
			Name: "projects/sample",
		}
		err = registryClient.DeleteProject(ctx, req)
		check(t, "Failed to delete sample project: %+v", err)
	}
	// Create the sample project.
	{
		req := &rpc.CreateProjectRequest{
			ProjectId: "sample",
		}
		project, err := registryClient.CreateProject(ctx, req)
		check(t, "error creating project %s", err)
		if project.GetName() != "projects/sample" {
			t.Errorf("Invalid project name %s", project.GetName())
		}
	}
	// List the sample project. This should return exactly one result.
	{
		req := &rpc.ListProjectsRequest{
			Filter: "project_id == 'sample'",
		}
		count := 0
		it := registryClient.ListProjects(ctx, req)
		for {
			project, err := it.Next()
			if err == nil {
				if project.Name != "projects/sample" {
					t.Errorf("Invalid project name: %s", project.Name)
				}
				count++
			} else {
				break
			}
		}
		if count != 1 {
			t.Errorf("Invalid project count: %d", count)
		}
	}
	// Get the sample project.
	{
		req := &rpc.GetProjectRequest{
			Name: "projects/sample",
		}
		project, err := registryClient.GetProject(ctx, req)
		check(t, "error getting project %s", err)
		if project.Name != "projects/sample" {
			t.Errorf("Invalid project name: %s", project.Name)
		}
	}
	// Create the petstore product.
	{
		req := &rpc.CreateProductRequest{
			Parent:    "projects/sample",
			ProductId: "petstore",
		}
		_, err := registryClient.CreateProduct(ctx, req)
		check(t, "error creating product %s", err)
	}
	// Create the petstore 1.0.0 version.
	{
		req := &rpc.CreateVersionRequest{
			Parent:    "projects/sample/products/petstore",
			VersionId: "1.0.0",
		}
		_, err := registryClient.CreateVersion(ctx, req)
		check(t, "error creating version %s", err)
	}
	// Upload the petstore 1.0.0 OpenAPI spec.
	{
		buf, err := readAndGZipFile("sample/petstore/1.0.0/openapi.yaml@r0")
		check(t, "error reading spec", err)
		req := &rpc.CreateSpecRequest{
			Parent: "projects/sample/products/petstore/versions/1.0.0",
			SpecId: "openapi.yaml",
			Spec: &rpc.Spec{
				Style:    "openapi/v3+gzip",
				Contents: buf.Bytes(),
			},
		}
		_, err = registryClient.CreateSpec(ctx, req)
		check(t, "error creating spec %s", err)
	}
	// Update the OpenAPI spec three times with different revisions.
	for _, filename := range []string{
		"sample/petstore/1.0.0/openapi.yaml@r1",
		"sample/petstore/1.0.0/openapi.yaml@r2",
		"sample/petstore/1.0.0/openapi.yaml@r3",
	} {
		buf, err := readAndGZipFile(filename)
		check(t, "error reading spec", err)
		req := &rpc.UpdateSpecRequest{
			Spec: &rpc.Spec{
				Name:     "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml",
				Contents: buf.Bytes(),
			},
		}
		_, err = registryClient.UpdateSpec(ctx, req)
		check(t, "error updating spec %s", err)
	}
	// List the spec revisions.
	{
		revisionIDs = listAllSpecRevisionIDs(ctx, registryClient)
		if len(revisionIDs) != 4 {
			t.Errorf("Incorrect revision count: %d", len(revisionIDs))
		}
	}
	// Check the hash of the original revision.
	if len(revisionIDs) > 0 {
		originalRevisionID = revisionIDs[len(revisionIDs)-1]
		req := &rpc.GetSpecRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml" + "@" + originalRevisionID,
		}
		spec, err := registryClient.GetSpec(ctx, req)
		check(t, "error getting spec %s", err)
		originalHash = spec.Hash
		// compute the hash of the original file
		buf, err := readAndGZipFile("sample/petstore/1.0.0/openapi.yaml@r0")
		check(t, "error reading spec", err)
		hash2 := hashForBytes(buf.Bytes())
		if originalHash != hash2 {
			t.Errorf("Hash mismatch %s != %s", originalHash, hash2)
		}
	}
	// List specs; there should be only one.
	{
		specs = listAllSpecs(ctx, registryClient)
		if len(specs) != 1 {
			t.Errorf("Incorrect spec count: %d", len(specs))
		}
	}
	// tag a spec revision
	{
		req := &rpc.TagSpecRevisionRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml" + "@" + originalRevisionID,
			Tag:  "og",
		}
		taggedSpec, err := registryClient.TagSpecRevision(ctx, req)
		check(t, "error tagging spec", err)
		if taggedSpec.Name != "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml@og" {
			t.Errorf("Incorrect name of tagged spec: %s", taggedSpec.Name)
		}
		if taggedSpec.Hash != originalHash {
			t.Errorf("Incorrect hash for tagged spec: %s", taggedSpec.Hash)
		}
	}
	// tag the tagged revision
	{
		req := &rpc.TagSpecRevisionRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml@og",
			Tag:  "first",
		}
		taggedSpec, err := registryClient.TagSpecRevision(ctx, req)
		check(t, "error tagging spec", err)
		if taggedSpec.Name != "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml@first" {
			t.Errorf("Incorrect name of tagged spec: %s", taggedSpec.Name)
		}
		if taggedSpec.Hash != originalHash {
			t.Errorf("Incorrect hash for tagged spec: %s", taggedSpec.Hash)
		}
	}
	// get a spec by its tag
	{
		req := &rpc.GetSpecRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml@first",
		}
		spec, err := registryClient.GetSpec(ctx, req)
		check(t, "error getting spec %s", err)
		if spec.Hash != originalHash {
			t.Errorf("Incorrect hash for spec retrieved by tag: %s", spec.Hash)
		}
	}
	// rollback a spec revision (this creates a new revision that's a copy)
	{
		req := &rpc.RollbackSpecRequest{
			Name:       "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml",
			RevisionId: "og",
		}
		spec, err := registryClient.RollbackSpec(ctx, req)
		check(t, "error rolling back spec %s", err)
		if spec.Hash != originalHash {
			t.Errorf("Incorrect hash for rolled-back spec: %s", spec.Hash)
		}
	}
	// List specs; there should be only one.
	{
		specs = listAllSpecs(ctx, registryClient)
		if len(specs) != 1 {
			t.Errorf("Incorrect spec count: %d", len(specs))
		}
	}
	// list spec revisions, there should now be five
	{
		revisionIDs = listAllSpecRevisionIDs(ctx, registryClient)
		if len(revisionIDs) != 5 {
			t.Errorf("Incorrect revision count: %d", len(revisionIDs))
		}
	}
	// delete a spec revision
	{
		req := &rpc.DeleteSpecRevisionRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml@og",
		}
		err := registryClient.DeleteSpecRevision(ctx, req)
		check(t, "error deleting spec revision %s", err)
	}
	// list specs, there should be only one
	{
		specs = listAllSpecs(ctx, registryClient)
		if len(specs) != 1 {
			t.Errorf("Incorrect spec count: %d", len(specs))
		}
	}
	// list spec revisions, there should be four
	{
		revisionIDs = listAllSpecRevisionIDs(ctx, registryClient)
		if len(revisionIDs) != 4 {
			t.Errorf("Incorrect revision count: %d", len(revisionIDs))
		}
	}
	// delete the spec
	{
		req := &rpc.DeleteSpecRequest{
			Name: "projects/sample/products/petstore/versions/1.0.0/specs/openapi.yaml",
		}
		err := registryClient.DeleteSpec(ctx, req)
		check(t, "error deleting spec %s", err)
	}
	// list spec revisions, there should be none
	{
		revisionIDs = listAllSpecRevisionIDs(ctx, registryClient)
		if len(revisionIDs) != 0 {
			t.Errorf("Incorrect revision count: %d", len(revisionIDs))
		}
	}
}
<|endoftext|>


<commit_before>//go:generate go run gen.go

package version

// Version tells us the app version string
const Version = "3.1.1+admin-api"
<commit_msg>version: unhack branch info<commit_after>//go:generate go run gen.go

package version

// Version tells us the app version string
const Version = "3.1.1"
<|endoftext|>


<commit_before>package testutils

func NewTestWriter(writeString *string) *TestWriter {
	var writer TestWriter
	writer.wrote = writeString
	return &writer
}

type TestWriter struct {
	wrote *string
}

func (self TestWriter) Write(p []byte) (n int, err error) {
	if self.wrote != nil {
		*self.wrote = string(p)
	}

	return len(p), nil
}

type TestReader struct {
	ToRead string
}

func (self TestReader) Read(p []byte) (n int, err error) {
	for i := 0; i < len(self.ToRead); i++ {
		p[i] = self.ToRead[i]
	}

	p[len(self.ToRead)] = '\n'

	return len(self.ToRead) + 1, nil
}

type TestReadWriter struct {
	TestReader
	TestWriter
}

// vim: nocindent
<commit_msg>Added some utility functions to testutils<commit_after>package testutils

import (
	"reflect"
	"regexp"
	"strings"
	"testing"
)

// import "fmt"

func NewTestWriter(writeString *string) *TestWriter {
	var writer TestWriter
	writer.wrote = writeString
	return &writer
}

type TestWriter struct {
	wrote *string
}

func (self TestWriter) Write(p []byte) (n int, err error) {
	if self.wrote != nil {
		*self.wrote = string(p)
	}

	return len(p), nil
}

type TestReader struct {
	ToRead string
}

func (self TestReader) Read(p []byte) (n int, err error) {
	for i := 0; i < len(self.ToRead); i++ {
		p[i] = self.ToRead[i]
	}

	p[len(self.ToRead)] = '\n'

	return len(self.ToRead) + 1, nil
}

type TestReadWriter struct {
	TestReader
	TestWriter
}

func TestSettersAndGetters(object interface{}, t *testing.T) bool {
	objType := reflect.TypeOf(object)

	regex, _ := regexp.Compile("^Get(.+)")

	getterToSetter := make(map[string]string)

	for i := 0; i < objType.NumMethod(); i++ {
		method := objType.Method(i)

		findMatchingFunctions := func(prefix1, prefix2 string) string {
			if strings.HasPrefix(method.Name, prefix1) {
				result := regex.FindStringSubmatch(method.Name)

				if result != nil {
					pairName := "Set" + result[1]
					_, found := objType.MethodByName(pairName)

					if !found {
						t.Logf("Unable to find matching setter/getter for %s.%s", objType.String(), method.Name)
						return ""
					}

					return pairName
				}
			}

			return ""
		}

		pairedMethodName := findMatchingFunctions("Get", "Set")
		if pairedMethodName != "" {
			getterToSetter[method.Name] = pairedMethodName
		}

		findMatchingFunctions("Set", "Get")
	}

	v := reflect.ValueOf(object)

	for g, s := range getterToSetter {
		getterValue := v.MethodByName(g)
		setterValue := v.MethodByName(s)

		getterType := getterValue.Type()
		setterType := setterValue.Type()

		if getterType.NumOut() != setterType.NumIn() {
			t.Errorf("In/out mismatch: %s:%v, %s:%v", g, getterType.NumOut(), s, setterType.NumIn())
		} else {
			vals := make([]reflect.Value, setterType.NumIn())

			for i := 0; i < len(vals); i++ {
				inType := setterType.In(i)
				t.Log("inType:", inType)
				vals[i] = reflect.New(inType)
			}

			setterValue.Call(vals)
		}
	}

	return true
}

func Assert(condition bool, t *testing.T, failMessage ...interface{}) {
	if !condition {
		t.Error(failMessage...)
	}
}

// vim: nocindent
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/Sirupsen/logrus"

	"github.com/mozilla/tls-observatory/config"
	"github.com/mozilla/tls-observatory/connection"
	pg "github.com/mozilla/tls-observatory/database"
	"github.com/mozilla/tls-observatory/logger"
	"github.com/mozilla/tls-observatory/worker"
)

var db *pg.DB
var log = logger.GetLogger()

var activeScanners int = 0

func main() {
	var (
		cfgFile, cipherscan string
		debug               bool
	)
	flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/scanner.cfg", "Input file csv format")
	flag.StringVar(&cipherscan, "b", "/opt/cipherscan/cipherscan", "Cipherscan binary location")
	flag.BoolVar(&debug, "debug", false, "Set debug logging")
	flag.Parse()

	if debug {
		logger.SetLevelToDebug()
	}

	conf, err := config.Load(cfgFile)
	if err != nil {
		log.Fatal(fmt.Sprintf("Failed to load configuration: %v", err))
	}
	if !conf.General.Enable && os.Getenv("TLSOBS_SCANNER_ENABLE") != "on" {
		log.Fatal("Scanner is disabled in configuration")
	}

	_, err = os.Stat(cipherscan)
	if err != nil {
		log.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Error("Could not locate cipherscan executable. TLS connection capabilities will not be available.")
	}

	// increase the n
	runtime.GOMAXPROCS(conf.General.MaxProc)

	dbtls := "disable"
	if conf.General.PostgresUseTLS {
		dbtls = "verify-full"
	}
	db, err = pg.RegisterConnection(
		conf.General.PostgresDB,
		conf.General.PostgresUser,
		conf.General.PostgresPass,
		conf.General.Postgres,
		dbtls)
	defer db.Close()
	if err != nil {
		log.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Fatal("Failed to connect to database")
	}
	db.SetMaxOpenConns(conf.General.MaxProc)
	db.SetMaxIdleConns(10)
	incomingScans := db.RegisterScanListener(conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, "disable")
	Setup(conf)

	for scanID := range incomingScans {
		// wait until we have an available scanner
		for {
			if activeScanners >= conf.General.MaxProc {
				time.Sleep(time.Second)
			} else {
				break
			}
		}
		go scan(scanID, cipherscan)
	}
}

func scan(scanID int64, cipherscan string) {
	activeScanners++
	defer func() {
		activeScanners--
	}()
	log.WithFields(logrus.Fields{
		"scan_id": scanID,
	}).Info("Received new scan")

	db.Exec("UPDATE scans SET attempts = attempts + 1 WHERE id=$1", scanID)

	scan, err := db.GetScanByID(scanID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"error":   err.Error(),
		}).Error("Could not find/decode scan")
		return
	}
	var completion int

	// Retrieve the certificate from the target
	certID, trustID, err := handleCert(scan.Target)
	if err != nil {
		db.Exec("UPDATE scans SET has_tls=FALSE, completion_perc=100 WHERE id=$1", scanID)
		log.WithFields(logrus.Fields{
			"scan_id":     scanID,
			"scan_Target": scan.Target,
			"error":       err.Error(),
		}).Error("Could not get certificate info")
		return
	}
	log.WithFields(logrus.Fields{
		"scan_id":  scanID,
		"cert_id":  certID,
		"trust_id": trustID,
	}).Debug("Retrieved certs")

	isTrustValid, err := db.IsTrustValid(trustID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"cert_id": certID,
			"error":   err.Error(),
		}).Error("Could not get if trust is valid")
		return
	}
	completion += 20
	_, err = db.Exec(`UPDATE scans
			SET cert_id=$1, trust_id=$2, has_tls=TRUE, is_valid=$3, completion_perc=$4
			WHERE id=$5`, certID, trustID, isTrustValid, completion, scanID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"cert_id": certID,
			"error":   err.Error(),
		}).Error("Could not update scans for cert")
		return
	}

	// Cipherscan the target
	js, err := connection.Connect(scan.Target, cipherscan)
	if err != nil {
		err, ok := err.(connection.NoTLSConnErr)
		if ok {
			//does not implement TLS
			db.Exec("UPDATE scans SET has_tls=FALSE, completion_perc=100 WHERE id=$1", scanID)
		} else {
			log.WithFields(logrus.Fields{
				"scan_id": scanID,
				"error":   err.Error(),
			}).Error("Could not get TLS connection info")
		}
		return
	}
	completion += 20
	_, err = db.Exec("UPDATE scans SET conn_info=$1, completion_perc=$2 WHERE id=$3",
		js, completion, scanID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"error":   err.Error(),
		}).Error("Could not update connection information for scan")
	}

	// Prepare worker input
	cert, err := db.GetCertByID(certID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"cert_id": certID,
		}).Error("Could not get certificate from db to pass to workers")
		return
	}
	var conn_info connection.Stored
	err = json.Unmarshal(js, &conn_info)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
		}).Error("Could not parse connection info to pass to workers")
		return
	}
	workerInput := worker.Input{
		DBHandle:    db,
		Scanid:      scanID,
		Certificate: *cert,
		Connection:  conn_info,
	}
	// launch workers that evaluate the results
	resChan := make(chan worker.Result)
	totalWorkers := 0
	for _, wrkInfo := range worker.AvailableWorkers {
		go wrkInfo.Runner.(worker.Worker).Run(workerInput, resChan)
		totalWorkers++
	}
	log.WithFields(logrus.Fields{
		"scan_id": scanID,
		"count":   totalWorkers,
	}).Info("Running workers")

	// read the results from the results chan in a loop until all workers have ran or expired
	for endedWorkers := 0; endedWorkers < totalWorkers; endedWorkers++ {
		select {
		case <-time.After(30 * time.Second):
			log.WithFields(logrus.Fields{
				"scan_id": scanID,
			}).Error("Analysis workers timed out after 30 seconds")
			return
		case res := <-resChan:
			endedWorkers += endedWorkers
			completion = ((endedWorkers/totalWorkers)*60 + completion)
			log.WithFields(logrus.Fields{
				"scan_id":     scanID,
				"worker_name": res.WorkerName,
				"success":     res.Success,
				"result":      string(res.Result),
			}).Debug("Received results from worker")

			err = db.UpdateScanCompletionPercentage(scanID, completion)
			if err != nil {
				log.WithFields(logrus.Fields{
					"scan_id": scanID,
					"error":   err.Error(),
				}).Error("Could not update completion percentage")
				continue
			}
			if !res.Success {
				log.WithFields(logrus.Fields{
					"worker_name": res.WorkerName,
					"errors":      res.Errors,
				}).Error("Worker returned with errors")
				continue
			}
			_, err = db.Exec("INSERT INTO analysis(scan_id,worker_name,output) VALUES($1,$2,$3)",
				scanID, res.WorkerName, res.Result)
			if err != nil {
				log.WithFields(logrus.Fields{
					"scan_id": scanID,
					"error":   err.Error(),
				}).Error("Could not insert worker results in database")
				continue
			}
			log.WithFields(logrus.Fields{
				"scan_id":     scanID,
				"worker_name": res.WorkerName,
			}).Info("Results from worker stored in database")
		}
	}
	err = db.UpdateScanCompletionPercentage(scanID, 100)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"error":   err.Error(),
		}).Error("Could not update completion percentage")
	}
	return
}
<commit_msg>typo in tlsobs-scanner flag description<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/Sirupsen/logrus"

	"github.com/mozilla/tls-observatory/config"
	"github.com/mozilla/tls-observatory/connection"
	pg "github.com/mozilla/tls-observatory/database"
	"github.com/mozilla/tls-observatory/logger"
	"github.com/mozilla/tls-observatory/worker"
)

var db *pg.DB
var log = logger.GetLogger()

var activeScanners int = 0

func main() {
	var (
		cfgFile, cipherscan string
		debug               bool
	)
	flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/scanner.cfg", "Configuration file")
	flag.StringVar(&cipherscan, "b", "/opt/cipherscan/cipherscan", "Cipherscan binary location")
	flag.BoolVar(&debug, "debug", false, "Set debug logging")
	flag.Parse()

	if debug {
		logger.SetLevelToDebug()
	}

	conf, err := config.Load(cfgFile)
	if err != nil {
		log.Fatal(fmt.Sprintf("Failed to load configuration: %v", err))
	}
	if !conf.General.Enable && os.Getenv("TLSOBS_SCANNER_ENABLE") != "on" {
		log.Fatal("Scanner is disabled in configuration")
	}

	_, err = os.Stat(cipherscan)
	if err != nil {
		log.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Error("Could not locate cipherscan executable. TLS connection capabilities will not be available.")
	}

	// increase the n
	runtime.GOMAXPROCS(conf.General.MaxProc)

	dbtls := "disable"
	if conf.General.PostgresUseTLS {
		dbtls = "verify-full"
	}
	db, err = pg.RegisterConnection(
		conf.General.PostgresDB,
		conf.General.PostgresUser,
		conf.General.PostgresPass,
		conf.General.Postgres,
		dbtls)
	defer db.Close()
	if err != nil {
		log.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Fatal("Failed to connect to database")
	}
	db.SetMaxOpenConns(conf.General.MaxProc)
	db.SetMaxIdleConns(10)
	incomingScans := db.RegisterScanListener(conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, "disable")
	Setup(conf)

	for scanID := range incomingScans {
		// wait until we have an available scanner
		for {
			if activeScanners >= conf.General.MaxProc {
				time.Sleep(time.Second)
			} else {
				break
			}
		}
		go scan(scanID, cipherscan)
	}
}

func scan(scanID int64, cipherscan string) {
	activeScanners++
	defer func() {
		activeScanners--
	}()
	log.WithFields(logrus.Fields{
		"scan_id": scanID,
	}).Info("Received new scan")

	db.Exec("UPDATE scans SET attempts = attempts + 1 WHERE id=$1", scanID)

	scan, err := db.GetScanByID(scanID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"error":   err.Error(),
		}).Error("Could not find/decode scan")
		return
	}
	var completion int

	// Retrieve the certificate from the target
	certID, trustID, err := handleCert(scan.Target)
	if err != nil {
		db.Exec("UPDATE scans SET has_tls=FALSE, completion_perc=100 WHERE id=$1", scanID)
		log.WithFields(logrus.Fields{
			"scan_id":     scanID,
			"scan_Target": scan.Target,
			"error":       err.Error(),
		}).Error("Could not get certificate info")
		return
	}
	log.WithFields(logrus.Fields{
		"scan_id":  scanID,
		"cert_id":  certID,
		"trust_id": trustID,
	}).Debug("Retrieved certs")

	isTrustValid, err := db.IsTrustValid(trustID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"cert_id": certID,
			"error":   err.Error(),
		}).Error("Could not get if trust is valid")
		return
	}
	completion += 20
	_, err = db.Exec(`UPDATE scans
			SET cert_id=$1, trust_id=$2, has_tls=TRUE, is_valid=$3, completion_perc=$4
			WHERE id=$5`, certID, trustID, isTrustValid, completion, scanID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"cert_id": certID,
			"error":   err.Error(),
		}).Error("Could not update scans for cert")
		return
	}

	// Cipherscan the target
	js, err := connection.Connect(scan.Target, cipherscan)
	if err != nil {
		err, ok := err.(connection.NoTLSConnErr)
		if ok {
			//does not implement TLS
			db.Exec("UPDATE scans SET has_tls=FALSE, completion_perc=100 WHERE id=$1", scanID)
		} else {
			log.WithFields(logrus.Fields{
				"scan_id": scanID,
				"error":   err.Error(),
			}).Error("Could not get TLS connection info")
		}
		return
	}
	completion += 20
	_, err = db.Exec("UPDATE scans SET conn_info=$1, completion_perc=$2 WHERE id=$3",
		js, completion, scanID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"error":   err.Error(),
		}).Error("Could not update connection information for scan")
	}

	// Prepare worker input
	cert, err := db.GetCertByID(certID)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"cert_id": certID,
		}).Error("Could not get certificate from db to pass to workers")
		return
	}
	var conn_info connection.Stored
	err = json.Unmarshal(js, &conn_info)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
		}).Error("Could not parse connection info to pass to workers")
		return
	}
	workerInput := worker.Input{
		DBHandle:    db,
		Scanid:      scanID,
		Certificate: *cert,
		Connection:  conn_info,
	}
	// launch workers that evaluate the results
	resChan := make(chan worker.Result)
	totalWorkers := 0
	for _, wrkInfo := range worker.AvailableWorkers {
		go wrkInfo.Runner.(worker.Worker).Run(workerInput, resChan)
		totalWorkers++
	}
	log.WithFields(logrus.Fields{
		"scan_id": scanID,
		"count":   totalWorkers,
	}).Info("Running workers")

	// read the results from the results chan in a loop until all workers have ran or expired
	for endedWorkers := 0; endedWorkers < totalWorkers; endedWorkers++ {
		select {
		case <-time.After(30 * time.Second):
			log.WithFields(logrus.Fields{
				"scan_id": scanID,
			}).Error("Analysis workers timed out after 30 seconds")
			return
		case res := <-resChan:
			endedWorkers += endedWorkers
			completion = ((endedWorkers/totalWorkers)*60 + completion)
			log.WithFields(logrus.Fields{
				"scan_id":     scanID,
				"worker_name": res.WorkerName,
				"success":     res.Success,
				"result":      string(res.Result),
			}).Debug("Received results from worker")

			err = db.UpdateScanCompletionPercentage(scanID, completion)
			if err != nil {
				log.WithFields(logrus.Fields{
					"scan_id": scanID,
					"error":   err.Error(),
				}).Error("Could not update completion percentage")
				continue
			}
			if !res.Success {
				log.WithFields(logrus.Fields{
					"worker_name": res.WorkerName,
					"errors":      res.Errors,
				}).Error("Worker returned with errors")
				continue
			}
			_, err = db.Exec("INSERT INTO analysis(scan_id,worker_name,output) VALUES($1,$2,$3)",
				scanID, res.WorkerName, res.Result)
			if err != nil {
				log.WithFields(logrus.Fields{
					"scan_id": scanID,
					"error":   err.Error(),
				}).Error("Could not insert worker results in database")
				continue
			}
			log.WithFields(logrus.Fields{
				"scan_id":     scanID,
				"worker_name": res.WorkerName,
			}).Info("Results from worker stored in database")
		}
	}
	err = db.UpdateScanCompletionPercentage(scanID, 100)
	if err != nil {
		log.WithFields(logrus.Fields{
			"scan_id": scanID,
			"error":   err.Error(),
		}).Error("Could not update completion percentage")
	}
	return
}
<|endoftext|>


<commit_before>package statement

import (
	"fmt"
	"regexp"
)

// Statement captures a single sql statement, ie INSERT INTO (...) VALUES (...);
type Statement struct {
	SQL  string
	Verb Verb // A statement type; INSERT, CREATE, etc
}

// Verb describes a type of sql statement, INSERT, CREATE, etc
type Verb int

const (
	// Create represents a Create Table statement
	Create Verb = iota
	// Insert represents an Insert statement
	Insert
)

func (v Verb) String() string {
	switch v {
	case Create:
		return "Create"
	case Insert:
		return "Insert"
	default:
		return ""
	}
}

// Table returns the name of the table that this statement operates upon.
func (s *Statement) Table() string {
	var reg *regexp.Regexp
	switch s.Verb {
	case Create:
		reg = regexp.MustCompile(`(?i)^CREATE TABLE \[?(?P<X>[A-Za-z]+)\]?`)
	case Insert:
		reg = regexp.MustCompile(`(?i)^INSERT INTO "?(?P<X>[A-Za-z]+)"?`)
	default:
		return ""
	}
	matches := reg.FindStringSubmatch(s.SQL)
	if len(matches) == 0 {
		return ""
	}
	// The first match should be *everything* from the regex, and the second
	// match should be only the capture group.
	return matches[1]
}

// Dependencies determines the foreign key dependencies of the table if the
// statement is a Create Statement
func (s *Statement) Dependencies() []string {
	if s.Verb != Create {
		return []string{}
	}

	reg := regexp.MustCompile(`(?i)REFERENCES \[?"?(?P<X>[A-Z]+)`)
	matches := reg.FindAllStringSubmatch(s.SQL, -1)
	var result []string
	for _, match := range matches {
		if len(match) == 0 {
			continue
		}
		if s.Table() != match[1] {
			result = append(result, match[1])
		}
	}

	return result
}

// Statements type is needed to satisfy the sort interface
type Statements []Statement

func (slice Statements) Len() int {
	return len(slice)
}

func (slice Statements) Less(i, j int) bool {
	return slice[i].Verb < slice[j].Verb
}

func (slice Statements) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

// FromString constructs a Statement from a string
func FromString(s string) (*Statement, error) {
	if isCreate, err := regexp.MatchString("(?i)^CREATE TABLE ", s); isCreate && err == nil {
		return &Statement{SQL: s, Verb: Create}, nil
	}
	if isInsert, err := regexp.MatchString("(?i)^INSERT INTO ", s); isInsert && err == nil {
		return &Statement{SQL: s, Verb: Insert}, nil
	}
	return nil, fmt.Errorf("A Statement could not be constructed from \"%s\"", s)
}

func (s *Statement) String() string {
	return s.SQL
}
<commit_msg>Robustify Statement FromString<commit_after>package statement

import (
	"fmt"
	"regexp"
	"strings"
)

// Statement captures a single sql statement, ie INSERT INTO (...) VALUES (...);
type Statement struct {
	SQL  string
	Verb Verb // A statement type; INSERT, CREATE, etc
}

// Verb describes a type of sql statement, INSERT, CREATE, etc
type Verb int

const (
	// Create represents a Create Table statement
	Create Verb = iota
	// Insert represents an Insert statement
	Insert
)

func (v Verb) String() string {
	switch v {
	case Create:
		return "Create"
	case Insert:
		return "Insert"
	default:
		return ""
	}
}

// Table returns the name of the table that this statement operates upon.
func (s *Statement) Table() string {
	var reg *regexp.Regexp
	switch s.Verb {
	case Create:
		reg = regexp.MustCompile(`(?i)^CREATE TABLE \[?(?P<X>[A-Za-z]+)\]?`)
	case Insert:
		reg = regexp.MustCompile(`(?i)^INSERT INTO "?(?P<X>[A-Za-z]+)"?`)
	default:
		return ""
	}
	matches := reg.FindStringSubmatch(s.SQL)
	if len(matches) == 0 {
		return ""
	}
	// The first match should be *everything* from the regex, and the second
	// match should be only the capture group.
	return matches[1]
}

// Dependencies determines the foreign key dependencies of the table if the
// statement is a Create Statement
func (s *Statement) Dependencies() []string {
	if s.Verb != Create {
		return []string{}
	}

	reg := regexp.MustCompile(`(?i)REFERENCES \[?"?(?P<X>[A-Z]+)`)
	matches := reg.FindAllStringSubmatch(s.SQL, -1)
	var result []string
	for _, match := range matches {
		if len(match) == 0 {
			continue
		}
		if s.Table() != match[1] {
			result = append(result, match[1])
		}
	}

	return result
}

// Statements type is needed to satisfy the sort interface
type Statements []Statement

func (slice Statements) Len() int {
	return len(slice)
}

func (slice Statements) Less(i, j int) bool {
	return slice[i].Verb < slice[j].Verb
}

func (slice Statements) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

// FromString constructs a Statement from a string
func FromString(s string) (*Statement, error) {
	s = strings.Trim(s, "\r\n\t ")
	if isCreate, err := regexp.MatchString("(?i)^CREATE TABLE ", s); isCreate && err == nil {
		return &Statement{SQL: s, Verb: Create}, nil
	}
	if isInsert, err := regexp.MatchString("(?i)^INSERT INTO ", s); isInsert && err == nil {
		return &Statement{SQL: s, Verb: Insert}, nil
	}
	return nil, fmt.Errorf("A Statement could not be constructed from \"%s\"", s)
}

func (s *Statement) String() string {
	return s.SQL
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package backend

import (
	"fmt"
	"net/http"
	"strings"

	"github.com/webx-top/com"
	"github.com/webx-top/echo"
	"github.com/webx-top/echo/handler/pprof"
	"github.com/webx-top/echo/middleware"
	"github.com/webx-top/echo/middleware/language"
	"github.com/webx-top/echo/middleware/render"
	"github.com/webx-top/echo/middleware/render/driver"
	"github.com/webx-top/echo/middleware/session"
	"github.com/webx-top/echo/subdomains"

	"github.com/admpub/log"
	"github.com/admpub/nging/application/cmd/event"
	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/config"
	ngingMW "github.com/admpub/nging/application/middleware"
)

const (
	DefaultTemplateDir   = `./template/backend`
	DefaultAssetsDir     = `./public/assets`
	DefaultAssetsURLPath = `/public/assets/backend`
)

var (
	TemplateDir      = DefaultTemplateDir //模板文件夹
	AssetsDir        = DefaultAssetsDir   //素材文件夹
	AssetsURLPath    = DefaultAssetsURLPath
	DefaultAvatarURL = AssetsURLPath + `/images/user_128.png`
	RendererDo       = func(driver.Driver) {}
	ParseStrings     = map[string]string{}
	ParseStringFuncs = map[string]func() string{}
	SkippedGzipPaths = map[string]bool{}
	GzipSkipper      = func(skippedPaths map[string]bool) func(c echo.Context) bool {
		return func(c echo.Context) bool {
			upath := c.Request().URL().Path()
			skipped, _ := skippedPaths[upath]
			return skipped
		}
	}
	DefaultLocalHostNames = []string{
		`127.0.0.1`, `localhost`,
	}
)

func MakeSubdomains(domain string, appends []string) string {
	domainList := strings.Split(domain, `,`)
	domain = domainList[0]
	if pos := strings.Index(domain, `://`); pos > 0 {
		pos += 3
		if pos < len(domain) {
			domain = domain[pos:]
		} else {
			domain = ``
		}
	}
	var myPort string
	domain, myPort = com.SplitHost(domain)
	if len(myPort) == 0 && len(domainList) > 1 {
		_, myPort = com.SplitHost(domainList[1])
	}
	port := fmt.Sprintf("%d", config.DefaultCLIConfig.Port)
	newDomainList := []string{}
	if !com.InSlice(domain+`:`+port, domainList) {
		newDomainList = append(newDomainList, domain+`:`+port)
	}
	if myPort == port {
		myPort = ``
	}
	if len(myPort) > 0 {
		if !com.InSlice(domain+`:`+myPort, domainList) {
			newDomainList = append(newDomainList, domain+`:`+myPort)
		}
	}
	for _, hostName := range appends {
		if hostName == domain {
			continue
		}
		if !com.InSlice(hostName+`:`+port, domainList) {
			newDomainList = append(newDomainList, hostName+`:`+port)
		}
		if len(myPort) > 0 {
			if !com.InSlice(hostName+`:`+myPort, domainList) {
				newDomainList = append(newDomainList, hostName+`:`+myPort)
			}
		}
	}
	if len(newDomainList) > 0 {
		domainList = append(domainList, newDomainList...)
	}
	return strings.Join(domainList, `,`)
}

func init() {
	echo.Set(`BackendPrefix`, handler.BackendPrefix)
	echo.Set(`GlobalPrefix`, handler.GlobalPrefix)
	event.OnStart(0, func() {
		handler.GlobalPrefix = echo.String(`GlobalPrefix`)
		handler.BackendPrefix = echo.String(`BackendPrefix`)
		handler.FrontendPrefix = echo.String(`FrontendPrefix`)
		ngingMW.DefaultAvatarURL = DefaultAssetsURLPath
		e := handler.Echo()
		e.SetPrefix(handler.GlobalPrefix)
		handler.SetRootGroup(handler.BackendPrefix)
		subdomains.Default.Default = `backend`
		subdomains.Default.Boot = `backend`
		domainName := subdomains.Default.Default
		backendDomain := config.DefaultCLIConfig.BackendDomain
		if len(backendDomain) > 0 {
			domainName += `@` + MakeSubdomains(backendDomain, DefaultLocalHostNames)
		}
		subdomains.Default.Add(domainName, e)

		e.Use(middleware.Log(), middleware.Recover())
		skippedGzipPaths := map[string]bool{
			e.Prefix() + `/server/cmdSend/info`:      true,
			e.Prefix() + `/download/progress/info`:   true,
			e.Prefix() + `/debug/pprof/`:             true,
			e.Prefix() + `/debug/pprof/allocs`:       true,
			e.Prefix() + `/debug/pprof/block`:        true,
			e.Prefix() + `/debug/pprof/cmdline`:      true,
			e.Prefix() + `/debug/pprof/goroutine`:    true,
			e.Prefix() + `/debug/pprof/heap`:         true,
			e.Prefix() + `/debug/pprof/mutex`:        true,
			e.Prefix() + `/debug/pprof/profile`:      true,
			e.Prefix() + `/debug/pprof/threadcreate`: true,
			e.Prefix() + `/debug/pprof/trace`:        true,
		}
		for k, v := range skippedGzipPaths {
			SkippedGzipPaths[k] = v
		}
		e.Use(middleware.Gzip(&middleware.GzipConfig{
			Skipper: GzipSkipper(SkippedGzipPaths),
		}))
		e.Use(func(h echo.Handler) echo.HandlerFunc {
			return func(c echo.Context) error {
				c.Response().Header().Set(`Server`, event.SoftwareName+`/`+config.Version.Number)
				return h.Handle(c)
			}
		})

		// 注册静态资源文件(网站素材文件)
		e.Use(event.StaticMW) //打包的静态资源
		// 上传文件资源(改到manager中用File函数实现)
		// e.Use(middleware.Static(&middleware.StaticOptions{
		// 	Root: helper.UploadDir,
		// 	Path: helper.UploadURLPath,
		// }))

		// 启用session
		e.Use(session.Middleware(config.SessionOptions))
		// 启用多语言支持
		config.DefaultConfig.Language.SetFSFunc(event.LangFSFunc)
		e.Use(language.New(&config.DefaultConfig.Language).Middleware())

		// 启用Validation
		e.Use(middleware.Validate(echo.NewValidation))

		// 事物支持
		e.Use(ngingMW.Tansaction())
		// 注册模板引擎
		renderOptions := &render.Config{
			TmplDir: TemplateDir,
			Engine:  `standard`,
			ParseStrings: map[string]string{
				`__ASSETS__`: AssetsURLPath,
				`__TMPL__`:   TemplateDir,
			},
			ParseStringFuncs: map[string]func() string{
				`__BACKEND__`:  func() string { return subdomains.Default.URL(handler.BackendPrefix, `backend`) },
				`__FRONTEND__`: func() string { return subdomains.Default.URL(handler.FrontendPrefix, `frontend`) },
			},
			DefaultHTTPErrorCode: http.StatusOK,
			Reload:               true,
			ErrorPages:           config.DefaultConfig.Sys.ErrorPages,
		}
		if ParseStrings != nil {
			for key, val := range ParseStrings {
				renderOptions.ParseStrings[key] = val
			}
		}
		if ParseStringFuncs != nil {
			for key, val := range ParseStringFuncs {
				renderOptions.ParseStringFuncs[key] = val
			}
		}
		if RendererDo != nil {
			renderOptions.AddRendererDo(RendererDo)
		}
		renderOptions.AddFuncSetter(ngingMW.ErrorPageFunc)
		renderOptions.ApplyTo(e, event.BackendTmplMgr)
		//RendererDo(renderOptions.Renderer())
		echo.On(`clearCache`, func(_ echo.H) error {
			log.Debug(`clear: Backend Template Object Cache`)
			renderOptions.Renderer().ClearCache()
			return nil
		})
		e.Get(`/favicon.ico`, event.FaviconHandler)
		if event.Develop {
			pprof.Wrap(e)
		}
		Initialize()
	})
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package backend

import (
	"fmt"
	"net/http"
	"strings"

	"github.com/webx-top/com"
	"github.com/webx-top/echo"
	"github.com/webx-top/echo/handler/pprof"
	"github.com/webx-top/echo/middleware"
	"github.com/webx-top/echo/middleware/language"
	"github.com/webx-top/echo/middleware/render"
	"github.com/webx-top/echo/middleware/render/driver"
	"github.com/webx-top/echo/middleware/session"
	"github.com/webx-top/echo/subdomains"

	"github.com/admpub/log"
	"github.com/admpub/nging/application/cmd/event"
	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/config"
	ngingMW "github.com/admpub/nging/application/middleware"
)

const (
	DefaultTemplateDir   = `./template/backend`
	DefaultAssetsDir     = `./public/assets`
	DefaultAssetsURLPath = `/public/assets/backend`
)

var (
	TemplateDir      = DefaultTemplateDir //模板文件夹
	AssetsDir        = DefaultAssetsDir   //素材文件夹
	AssetsURLPath    = DefaultAssetsURLPath
	DefaultAvatarURL = AssetsURLPath + `/images/user_128.png`
	RendererDo       = func(driver.Driver) {}
	ParseStrings     = map[string]string{}
	ParseStringFuncs = map[string]func() string{}
	SkippedGzipPaths = map[string]bool{}
	GzipSkipper      = func(skippedPaths map[string]bool) func(c echo.Context) bool {
		return func(c echo.Context) bool {
			upath := c.Request().URL().Path()
			skipped, _ := skippedPaths[upath]
			return skipped
		}
	}
	DefaultLocalHostNames = []string{
		`127.0.0.1`, `localhost`,
	}
)

func MakeSubdomains(domain string, appends []string) string {
	domainList := strings.Split(domain, `,`)
	domain = domainList[0]
	if pos := strings.Index(domain, `://`); pos > 0 {
		pos += 3
		if pos < len(domain) {
			domain = domain[pos:]
		} else {
			domain = ``
		}
	}
	var myPort string
	domain, myPort = com.SplitHost(domain)
	if len(myPort) == 0 && len(domainList) > 1 {
		_, myPort = com.SplitHost(domainList[1])
	}
	port := fmt.Sprintf("%d", config.DefaultCLIConfig.Port)
	newDomainList := []string{}
	if !com.InSlice(domain+`:`+port, domainList) {
		newDomainList = append(newDomainList, domain+`:`+port)
	}
	if myPort == port {
		myPort = ``
	}
	if len(myPort) > 0 {
		if !com.InSlice(domain+`:`+myPort, domainList) {
			newDomainList = append(newDomainList, domain+`:`+myPort)
		}
	}
	for _, hostName := range appends {
		if hostName == domain {
			continue
		}
		if !com.InSlice(hostName+`:`+port, domainList) {
			newDomainList = append(newDomainList, hostName+`:`+port)
		}
		if len(myPort) > 0 {
			if !com.InSlice(hostName+`:`+myPort, domainList) {
				newDomainList = append(newDomainList, hostName+`:`+myPort)
			}
		}
	}
	if len(newDomainList) > 0 {
		domainList = append(domainList, newDomainList...)
	}
	return strings.Join(domainList, `,`)
}

func init() {
	echo.Set(`BackendPrefix`, handler.BackendPrefix)
	echo.Set(`GlobalPrefix`, handler.GlobalPrefix)
	event.OnStart(0, func() {
		handler.GlobalPrefix = echo.String(`GlobalPrefix`)
		handler.BackendPrefix = echo.String(`BackendPrefix`)
		handler.FrontendPrefix = echo.String(`FrontendPrefix`)
		ngingMW.DefaultAvatarURL = DefaultAssetsURLPath
		e := handler.Echo()
		e.SetPrefix(handler.GlobalPrefix)
		handler.SetRootGroup(handler.BackendPrefix)
		subdomains.Default.Default = `backend`
		subdomains.Default.Boot = `backend`
		domainName := subdomains.Default.Default
		backendDomain := config.DefaultCLIConfig.BackendDomain
		if len(backendDomain) > 0 {
			domainName += `@` + MakeSubdomains(backendDomain, DefaultLocalHostNames)
		}
		subdomains.Default.Add(domainName, e)

		e.Use(middleware.Log(), middleware.Recover())
		e.Use(middleware.Gzip(&middleware.GzipConfig{
			Skipper: GzipSkipper(SkippedGzipPaths),
		}))
		e.Use(func(h echo.Handler) echo.HandlerFunc {
			return func(c echo.Context) error {
				c.Response().Header().Set(`Server`, event.SoftwareName+`/`+config.Version.Number)
				return h.Handle(c)
			}
		})

		// 注册静态资源文件(网站素材文件)
		e.Use(event.StaticMW) //打包的静态资源
		// 上传文件资源(改到manager中用File函数实现)
		// e.Use(middleware.Static(&middleware.StaticOptions{
		// 	Root: helper.UploadDir,
		// 	Path: helper.UploadURLPath,
		// }))

		// 启用session
		e.Use(session.Middleware(config.SessionOptions))
		// 启用多语言支持
		config.DefaultConfig.Language.SetFSFunc(event.LangFSFunc)
		e.Use(language.New(&config.DefaultConfig.Language).Middleware())

		// 启用Validation
		e.Use(middleware.Validate(echo.NewValidation))

		// 事物支持
		e.Use(ngingMW.Tansaction())
		// 注册模板引擎
		renderOptions := &render.Config{
			TmplDir: TemplateDir,
			Engine:  `standard`,
			ParseStrings: map[string]string{
				`__ASSETS__`: AssetsURLPath,
				`__TMPL__`:   TemplateDir,
			},
			ParseStringFuncs: map[string]func() string{
				`__BACKEND__`:  func() string { return subdomains.Default.URL(handler.BackendPrefix, `backend`) },
				`__FRONTEND__`: func() string { return subdomains.Default.URL(handler.FrontendPrefix, `frontend`) },
			},
			DefaultHTTPErrorCode: http.StatusOK,
			Reload:               true,
			ErrorPages:           config.DefaultConfig.Sys.ErrorPages,
		}
		if ParseStrings != nil {
			for key, val := range ParseStrings {
				renderOptions.ParseStrings[key] = val
			}
		}
		if ParseStringFuncs != nil {
			for key, val := range ParseStringFuncs {
				renderOptions.ParseStringFuncs[key] = val
			}
		}
		if RendererDo != nil {
			renderOptions.AddRendererDo(RendererDo)
		}
		renderOptions.AddFuncSetter(ngingMW.ErrorPageFunc)
		renderOptions.ApplyTo(e, event.BackendTmplMgr)
		//RendererDo(renderOptions.Renderer())
		echo.On(`clearCache`, func(_ echo.H) error {
			log.Debug(`clear: Backend Template Object Cache`)
			renderOptions.Renderer().ClearCache()
			return nil
		})
		e.Get(`/favicon.ico`, event.FaviconHandler)
		if event.Develop {
			pprof.Wrap(e)
		}
		Initialize()
	})
}
<|endoftext|>


<commit_before>package app

import (
   "encoding/json"
   "fmt"
   "os"
   "os/signal"
   //"syscall"
   "sync"
   model "lessa/org/app"
)

// private options, exposed indirectly via the Builder interface
// options are embedded by builder and application to promote
// implementation reuse
// public members on the type buys us json marshalling support
type options struct {
   Option1 string
   Option2 int
   Option3 float64
   Option4 []string
   Option5 []int
   Option6 []float64
}

// builder has options
type builder struct {
   options
   jobs    []model.Job
}

// application has options
type application struct {
   options
   jobs    []model.Job
   done    chan bool
   sigs    chan os.Signal
}

// allow clients to install a default options builder
func InstallAppBuilderFunc() {
   model.SetAppBuilderFunc(defaultBuilder)
}

// pretty print
func (o options) String() string {
   js, _ := json.MarshalIndent(o, "", "   ")
   return string(js)
}

// dummy
func (a application) Run() error {

   fmt.Println()
   fmt.Println("[impl/application] Registering for specified signal types.")
   signal.Notify(a.sigs)

   fmt.Println("[impl/application] Setting up signal handling.")
   go a.waitForSignal()

   if len(a.jobs) > 0 {
      fmt.Println("[impl/application] Spawning jobs.")
      go a.runJobs()
   }

   fmt.Println("[impl/application] Waiting for jobs or a registered signal.")
   <- a.done

   fmt.Println("[impl/application] Received and processed signal.")
   return nil
}

func (a application) runJobs() {

   wg := sync.WaitGroup{}

   for _, each := range a.jobs {

      // add the job to the list
      wg.Add(1)

	  // run each job in the background
	  //
	  // Note: we pass a copy of the 'each' variable as a parameter to the goroutine
	  // to avoid non-determinism; using the value of 'each' directly in the body of
	  // the goroutine would cause it to be evaluated only when effectively used; by
	  // then it is possible that the value of 'each' has already been modified
	  go func(job model.Job) {

	     defer wg.Done()
         job(a)
      }(each)
   }

   wg.Wait()

   a.done <- true
}

func (a application) waitForSignal() {

   select {
      // waiting for a registered signal
      case sig := <-a.sigs:

         fmt.Println()
         fmt.Println("[impl/application] Signal received:", sig)

         // cleaning up allocated resources
         a.cleanup()

         // releasing the runnable
         a.done <- true

         // remove from select
         a.sigs = nil
   }
}

func (a application) cleanup() error {

   fmt.Println("[impl/application] Cleaning up prior to stopping.")
   return nil
}

func (a application) Stop() error {

   fmt.Println("[impl/application] Sending an interrupt signal.")
   a.sigs <- os.Interrupt

   fmt.Println("[impl/application] Stop completed.")
   return nil
}

// default options used by the default builder
func defaultoptions() options {
   return options{
      Option1: "default",
      Option2: 666,
      Option3: 666.666,
      Option4: []string{"default1","default2"},
      Option5: []int{666, 999},
      Option6: []float64{666.999, 999.666},
   }
}

// default builder
func defaultBuilder() model.Builder {
   return builder{
      options: defaultoptions(),
   }
}

// creates an application by passing a copy of the builder's options
func (b builder) Build() model.Application {
   return application {
      options: b.options,
      jobs:    b.jobs,
      done:    make(chan bool, 1),
      sigs:    make(chan os.Signal, 1),
   }
}

// update the internal jobs
func (b builder) WithJobs(jobs... model.Job) model.Builder {
   b.jobs = jobs
   return b
}

// update the internal options
func (b builder) WithOption1(val string) model.Builder {
   b.Option1 = val
   return b
}

// update the internal options
func (b builder) WithOption2(val int) model.Builder {
   b.Option2 = val
   return b
}

// update the internal options
func (b builder) WithOption3(val float64) model.Builder {
   b.Option3 = val
   return b
}

// update the internal options
func (b builder) WithOption4(val []string) model.Builder {
   b.Option4 = val
   return b
}

// update the internal options
func (b builder) WithOption5(val []int) model.Builder {
   b.Option5 = val
   return b
}

// update the internal options
func (b builder) WithOption6(val []float64) model.Builder {
   b.Option6 = val
   return b
}
<commit_msg>Format/tabbing.<commit_after>package app

import (
   "encoding/json"
   "fmt"
   "os"
   "os/signal"
   //"syscall"
   "sync"
   model "lessa/org/app"
)

// private options, exposed indirectly via the Builder interface
// options are embedded by builder and application to promote
// implementation reuse
// public members on the type buys us json marshalling support
type options struct {
   Option1 string
   Option2 int
   Option3 float64
   Option4 []string
   Option5 []int
   Option6 []float64
}

// builder has options
type builder struct {
   options
   jobs    []model.Job
}

// application has options
type application struct {
   options
   jobs    []model.Job
   done    chan bool
   sigs    chan os.Signal
}

// allow clients to install a default options builder
func InstallAppBuilderFunc() {
   model.SetAppBuilderFunc(defaultBuilder)
}

// pretty print
func (o options) String() string {
   js, _ := json.MarshalIndent(o, "", "   ")
   return string(js)
}

// dummy
func (a application) Run() error {

   fmt.Println()
   fmt.Println("[impl/application] Registering for specified signal types.")
   signal.Notify(a.sigs)

   fmt.Println("[impl/application] Setting up signal handling.")
   go a.waitForSignal()

   if len(a.jobs) > 0 {
      fmt.Println("[impl/application] Spawning jobs.")
      go a.runJobs()
   }

   fmt.Println("[impl/application] Waiting for jobs or a registered signal.")
   <- a.done

   fmt.Println("[impl/application] Received and processed signal.")
   return nil
}

func (a application) runJobs() {

   wg := sync.WaitGroup{}

   for _, each := range a.jobs {

      // add the job to the list
      wg.Add(1)

	  // run each job in the background
	  //
	  // Note: we pass a copy of the 'each' variable as a parameter to the goroutine
	  // to avoid non-determinism; using the value of 'each' directly in the body of
	  // the goroutine would cause it to be evaluated only when effectively used; by
	  // then it is possible that the value of 'each' has already been modified
	  go func(job model.Job) {

         defer wg.Done()
         job(a)
      }(each)
   }

   wg.Wait()

   a.done <- true
}

func (a application) waitForSignal() {

   select {
      // waiting for a registered signal
      case sig := <-a.sigs:

         fmt.Println()
         fmt.Println("[impl/application] Signal received:", sig)

         // cleaning up allocated resources
         a.cleanup()

         // releasing the runnable
         a.done <- true

         // remove from select
         a.sigs = nil
   }
}

func (a application) cleanup() error {

   fmt.Println("[impl/application] Cleaning up prior to stopping.")
   return nil
}

func (a application) Stop() error {

   fmt.Println("[impl/application] Sending an interrupt signal.")
   a.sigs <- os.Interrupt

   fmt.Println("[impl/application] Stop completed.")
   return nil
}

// default options used by the default builder
func defaultoptions() options {
   return options{
      Option1: "default",
      Option2: 666,
      Option3: 666.666,
      Option4: []string{"default1","default2"},
      Option5: []int{666, 999},
      Option6: []float64{666.999, 999.666},
   }
}

// default builder
func defaultBuilder() model.Builder {
   return builder{
      options: defaultoptions(),
   }
}

// creates an application by passing a copy of the builder's options
func (b builder) Build() model.Application {
   return application {
      options: b.options,
      jobs:    b.jobs,
      done:    make(chan bool, 1),
      sigs:    make(chan os.Signal, 1),
   }
}

// update the internal jobs
func (b builder) WithJobs(jobs... model.Job) model.Builder {
   b.jobs = jobs
   return b
}

// update the internal options
func (b builder) WithOption1(val string) model.Builder {
   b.Option1 = val
   return b
}

// update the internal options
func (b builder) WithOption2(val int) model.Builder {
   b.Option2 = val
   return b
}

// update the internal options
func (b builder) WithOption3(val float64) model.Builder {
   b.Option3 = val
   return b
}

// update the internal options
func (b builder) WithOption4(val []string) model.Builder {
   b.Option4 = val
   return b
}

// update the internal options
func (b builder) WithOption5(val []int) model.Builder {
   b.Option5 = val
   return b
}

// update the internal options
func (b builder) WithOption6(val []float64) model.Builder {
   b.Option6 = val
   return b
}
<|endoftext|>