<commit_before>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"k8s.io/apimachinery/pkg/runtime"
	kscheme "k8s.io/client-go/kubernetes/scheme"
	_ "k8s.io/client-go/plugin/pkg/client/auth"
	"k8s.io/klog"
	apireg "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
	ctrl "sigs.k8s.io/controller-runtime"

	certmgrscheme "github.com/jetstack/cert-manager/pkg/client/clientset/versioned/scheme"
	"github.com/jetstack/cert-manager/pkg/controller/cainjector"
	"github.com/jetstack/cert-manager/pkg/util"
)

var scheme = runtime.NewScheme()

func init() {
	kscheme.AddToScheme(scheme)
	certmgrscheme.AddToScheme(scheme)
	apireg.AddToScheme(scheme)
}

type InjectorControllerOptions struct {
	Namespace               string
	LeaderElect             bool
	LeaderElectionNamespace string
	LeaderElectionID        string

	StdOut io.Writer
	StdErr io.Writer
}

func (o *InjectorControllerOptions) AddFlags(fs *pflag.FlagSet) {
	fs.StringVar(&o.Namespace, "namespace", "", ""+
		"If set, this limits the scope of cert-manager to a single namespace and ClusterIssuers are disabled. "+
		"If not specified, all namespaces will be watched")
	fs.BoolVar(&o.LeaderElect, "leader-elect", true, ""+
		"If true, cert-manager will perform leader election between instances to ensure no more "+
		"than one instance of cert-manager operates at a time")
	fs.StringVar(&o.LeaderElectionNamespace, "leader-election-namespace", "", ""+
		"Namespace used to perform leader election (defaults to controller's namespace). "+
		"Only used if leader election is enabled")
	fs.StringVar(&o.LeaderElectionNamespace, "leader-election-id", "", ""+
		"Override the identifier to use in leader election.  Only used if leader election is enabled")
}

func NewInjectorControllerOptions(out, errOut io.Writer) *InjectorControllerOptions {
	o := &InjectorControllerOptions{
		StdOut: out,
		StdErr: errOut,
	}

	return o
}

// NewCommandStartInjectorController is a CLI handler for starting cert-manager
func NewCommandStartInjectorController(out, errOut io.Writer, stopCh <-chan struct{}) *cobra.Command {
	o := NewInjectorControllerOptions(out, errOut)

	cmd := &cobra.Command{
		Use:   "ca-injector-controller",
		Short: fmt.Sprintf("CA Injection Controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit),
		Long: `
cert-manager CA injector is a Kubernetes addon to automate the injection of CA data into
webhooks and APIServices from cert-manager certificates.

It will ensure that annotated webhooks and API services always have the correct
CA data from the referenced certificates, which can then be used to serve API
servers and webhook servers.`,

		// TODO: Refactor this function from this package
		Run: func(cmd *cobra.Command, args []string) {
			klog.Infof("starting ca-injector-controller %s (revision %s)", util.AppVersion, util.AppGitCommit)
			o.RunInjectorController(stopCh)
		},
	}

	flags := cmd.Flags()
	o.AddFlags(flags)

	return cmd
}

func (o InjectorControllerOptions) RunInjectorController(stopCh <-chan struct{}) {
	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
		Scheme:                  scheme,
		Namespace:               o.Namespace,
		LeaderElection:          o.LeaderElect,
		LeaderElectionNamespace: o.LeaderElectionNamespace,
		LeaderElectionID:        o.LeaderElectionID,
	})

	if err != nil {
		klog.Fatalf("error creating manager: %v", err)
	}

	// TODO(directxman12): enabled controllers for separate injectors?
	if err := cainjector.RegisterAll(mgr); err != nil {
		klog.Fatalf("error registering controllers: %v", err)
	}

	if err := mgr.Start(stopCh); err != nil {
		klog.Fatalf("error running manager: %v", err)
	}
}
<commit_msg>Update flag descriptions<commit_after>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"k8s.io/apimachinery/pkg/runtime"
	kscheme "k8s.io/client-go/kubernetes/scheme"
	_ "k8s.io/client-go/plugin/pkg/client/auth"
	"k8s.io/klog"
	apireg "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
	ctrl "sigs.k8s.io/controller-runtime"

	certmgrscheme "github.com/jetstack/cert-manager/pkg/client/clientset/versioned/scheme"
	"github.com/jetstack/cert-manager/pkg/controller/cainjector"
	"github.com/jetstack/cert-manager/pkg/util"
)

var scheme = runtime.NewScheme()

func init() {
	kscheme.AddToScheme(scheme)
	certmgrscheme.AddToScheme(scheme)
	apireg.AddToScheme(scheme)
}

type InjectorControllerOptions struct {
	Namespace               string
	LeaderElect             bool
	LeaderElectionNamespace string
	LeaderElectionID        string

	StdOut io.Writer
	StdErr io.Writer
}

func (o *InjectorControllerOptions) AddFlags(fs *pflag.FlagSet) {
	fs.StringVar(&o.Namespace, "namespace", "", ""+
		"If set, this limits the scope of cainjector to a single namespace. " +
		"If set, cainjector will not update resources with certificates outside of the "+
		"configured namespace.")
	fs.BoolVar(&o.LeaderElect, "leader-elect", true, ""+
		"If true, cainjector will perform leader election between instances to ensure no more "+
		"than one instance of cainjector operates at a time")
	fs.StringVar(&o.LeaderElectionNamespace, "leader-election-namespace", "", ""+
		"Namespace used to perform leader election (defaults to controller's namespace). "+
		"Only used if leader election is enabled")
	fs.StringVar(&o.LeaderElectionID, "leader-election-id", "", ""+
		"Override the identifier to use in leader election.  Only used if leader election is enabled")
}

func NewInjectorControllerOptions(out, errOut io.Writer) *InjectorControllerOptions {
	o := &InjectorControllerOptions{
		StdOut: out,
		StdErr: errOut,
	}

	return o
}

// NewCommandStartInjectorController is a CLI handler for starting cert-manager
func NewCommandStartInjectorController(out, errOut io.Writer, stopCh <-chan struct{}) *cobra.Command {
	o := NewInjectorControllerOptions(out, errOut)

	cmd := &cobra.Command{
		Use:   "ca-injector-controller",
		Short: fmt.Sprintf("CA Injection Controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit),
		Long: `
cert-manager CA injector is a Kubernetes addon to automate the injection of CA data into
webhooks and APIServices from cert-manager certificates.

It will ensure that annotated webhooks and API services always have the correct
CA data from the referenced certificates, which can then be used to serve API
servers and webhook servers.`,

		// TODO: Refactor this function from this package
		Run: func(cmd *cobra.Command, args []string) {
			klog.Infof("starting ca-injector-controller %s (revision %s)", util.AppVersion, util.AppGitCommit)
			o.RunInjectorController(stopCh)
		},
	}

	flags := cmd.Flags()
	o.AddFlags(flags)

	return cmd
}

func (o InjectorControllerOptions) RunInjectorController(stopCh <-chan struct{}) {
	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
		Scheme:                  scheme,
		Namespace:               o.Namespace,
		LeaderElection:          o.LeaderElect,
		LeaderElectionNamespace: o.LeaderElectionNamespace,
		LeaderElectionID:        o.LeaderElectionID,
	})

	if err != nil {
		klog.Fatalf("error creating manager: %v", err)
	}

	// TODO(directxman12): enabled controllers for separate injectors?
	if err := cainjector.RegisterAll(mgr); err != nil {
		klog.Fatalf("error registering controllers: %v", err)
	}

	if err := mgr.Start(stopCh); err != nil {
		klog.Fatalf("error running manager: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/Symantec/keymaster/lib/webapi/v0/proto"
	"github.com/pquerna/otp/totp"
)

func TestEncryptDecryptSuccess(t *testing.T) {
	state, passwdFile, err := setupValidRuntimeStateSigner()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(passwdFile.Name()) // clean up

	testSecret := "superSecret"
	cipherTexts, err := state.encryptWithPublicKeys([]byte(testSecret))
	if err != nil {
		t.Fatal(err)
	}
	plainTextBytes, err := state.decryptWithPublicKeys(cipherTexts)

	if err != nil {
		t.Fatal(err)
	}
	if string(plainTextBytes) != testSecret {
		t.Fatal("values do not match")
	}
}

func TestGenerateNewTOTPSuccess(t *testing.T) {
	state, passwdFile, err := setupValidRuntimeStateSigner()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(passwdFile.Name()) // clean up

	state.Config.Base.AllowedAuthBackendsForWebUI = append(state.Config.Base.AllowedAuthBackendsForWebUI, proto.AuthTypeU2F)

	state.signerPublicKeyToKeymasterKeys()

	dir, err := ioutil.TempDir("", "example")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up
	state.Config.Base.DataDirectory = dir
	err = initDB(state)
	if err != nil {
		t.Fatal(err)
	}

	/// End of setup

	req, err := http.NewRequest("GET", totpGeneratNewPath, nil)
	if err != nil {
		t.Fatal(err)
		//return nil, err
	}
	cookieVal, err := state.setNewAuthCookie(nil, "username", AuthTypeU2F)
	if err != nil {
		t.Fatal(err)
	}
	authCookie := http.Cookie{Name: authCookieName, Value: cookieVal}
	req.AddCookie(&authCookie)

	tokenRR, err := checkRequestHandlerCode(req, state.GenerateNewTOTP, http.StatusOK)
	if err != nil {
		t.Fatal(err)
	}
	resultAccessToken := newTOTPPageTemplateData{}
	body := tokenRR.Result().Body
	err = json.NewDecoder(body).Decode(&resultAccessToken)
	if err != nil {
		t.Fatal(err)
	}
	t.Logf("totpDataToken='%+v'", resultAccessToken)

	// now we validate
	otpValue, err := totp.GenerateCode(resultAccessToken.TOTPSecret, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	data := url.Values{}
	data.Set("OTP", otpValue)
	t.Logf("url_Data='%+v'", data)
	validateReq, err := http.NewRequest("POST", totpValidateNewPath, bytes.NewBufferString(data.Encode()))
	if err != nil {
		t.Fatal(err)
		//return nil, err
	}
	validateReq.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	validateReq.AddCookie(&authCookie)
	_, err = checkRequestHandlerCode(validateReq, state.validateNewTOTP, http.StatusFound)
	if err != nil {
		t.Fatal(err)
	}
	//now check auth against it
	now := time.Now()
	otpValueInt, err := strconv.Atoi(otpValue)
	if err != nil {
		t.Fatal(err)
	}
	valid, err := state.validateUserTOTP("username", otpValueInt, now)
	if err != nil {
		t.Fatal(err)
	}
	if !valid {
		t.Fatal("should have been valid")
	}
	// now we retry with same value and should fail
	valid, err = state.validateUserTOTP("username", otpValueInt, now)
	if err != nil {
		t.Fatal(err)
	}
	if valid {
		t.Fatal("should NOT have been valid")
	}

}

func setupTestStateWithTOTPSecret(t *testing.T, state *RuntimeState, cookieAuth int) (*http.Cookie, string, error) {

	authUser := "username"
	key, err := totp.Generate(totp.GenerateOpts{
		Issuer:      "keymaster-totp", //TODO: get the actual name
		AccountName: authUser,
	})
	if err != nil {
		t.Fatal(err)
	}
	profile, _, _, err := state.LoadUserProfile(authUser)
	if err != nil {
		t.Fatal(err)
	}
	encryptedKeys, err := state.encryptWithPublicKeys([]byte(key.Secret()))
	if err != nil {
		t.Fatal(err)
	}
	newTOTPAuthData := totpAuthData{
		CreatedAt:       time.Now(),
		EncryptedSecret: encryptedKeys,
		//ValidatorAddr:   r.RemoteAddr,
		Enabled: true,
	}
	newIndex := newTOTPAuthData.CreatedAt.Unix()
	profile.TOTPAuthData[newIndex] = &newTOTPAuthData

	err = state.SaveUserProfile(authUser, profile)
	if err != nil {
		t.Fatal(err)
	}
	cookieVal, err := state.setNewAuthCookie(nil, "username", cookieAuth)
	if err != nil {
		t.Fatal(err)
	}
	authCookie := http.Cookie{Name: authCookieName, Value: cookieVal}
	return &authCookie, key.Secret(), nil
}

func TestVerifyTOTPHandlerSuccess(t *testing.T) {
	state, passwdFile, err := setupValidRuntimeStateSigner()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(passwdFile.Name()) // clean up

	state.Config.Base.AllowedAuthBackendsForWebUI = append(state.Config.Base.AllowedAuthBackendsForWebUI, proto.AuthTypeU2F)

	state.signerPublicKeyToKeymasterKeys()

	dir, err := ioutil.TempDir("", "example-1")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up
	state.Config.Base.DataDirectory = dir
	err = initDB(state)
	if err != nil {
		t.Fatal(err)
	}

	/// End of setup
	authCookie, totpSecret, err := setupTestStateWithTOTPSecret(t, state, AuthTypeU2F)
	if err != nil {
		t.Fatal(err)
	}
	otpValue, err := totp.GenerateCode(totpSecret, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	data := url.Values{}
	data.Set("OTP", otpValue)
	t.Logf("i testverify url_Data='%+v'", data)
	verifyReq, err := http.NewRequest("POST", totpValidateNewPath, bytes.NewBufferString(data.Encode()))
	if err != nil {
		t.Fatal(err)
	}
	verifyReq.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	verifyReq.AddCookie(authCookie)
	_, err = checkRequestHandlerCode(verifyReq, state.verifyTOTPHandler, http.StatusFound)
	if err != nil {
		t.Fatal(err)
	}
}

func TestAuthTOTPHandlerSuccess(t *testing.T) {
	state, passwdFile, err := setupValidRuntimeStateSigner()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(passwdFile.Name()) // clean up

	state.Config.Base.AllowedAuthBackendsForWebUI = append(state.Config.Base.AllowedAuthBackendsForWebUI, proto.AuthTypeTOTP)

	state.signerPublicKeyToKeymasterKeys()

	dir, err := ioutil.TempDir("", "example")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up
	state.Config.Base.DataDirectory = dir
	err = initDB(state)
	if err != nil {
		t.Fatal(err)
	}
	/// End of setup
	authCookie, totpSecret, err := setupTestStateWithTOTPSecret(t, state, AuthTypePassword)
	if err != nil {
		t.Fatal(err)
	}

	otpValue, err := totp.GenerateCode(totpSecret, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	data := url.Values{}
	data.Set("OTP", otpValue)
	t.Logf("i testverify url_Data='%+v'", data)
	verifyReq, err := http.NewRequest("POST", totpValidateNewPath, bytes.NewBufferString(data.Encode()))
	if err != nil {
		t.Fatal(err)
	}
	verifyReq.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	verifyReq.AddCookie(authCookie)
	_, err = checkRequestHandlerCode(verifyReq, state.TOTPAuthHandler, http.StatusOK)
	if err != nil {
		t.Fatal(err)
	}
}

func TestTOTPTokenManagerHandlerUpdateSuccess(t *testing.T) {
	state, passwdFile, err := setupValidRuntimeStateSigner()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(passwdFile.Name()) // clean up

	state.Config.Base.AllowedAuthBackendsForWebUI = append(state.Config.Base.AllowedAuthBackendsForWebUI, proto.AuthTypePassword)

	state.signerPublicKeyToKeymasterKeys()

	dir, err := ioutil.TempDir("", "example")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up
	state.Config.Base.DataDirectory = dir
	err = initDB(state)
	if err != nil {
		t.Fatal(err)
	}
	//
	cookieVal, err := state.setNewAuthCookie(nil, "username", AuthTypeAny)
	if err != nil {
		t.Fatal(err)
	}
	//cookieReq.AddCookie(&authCookie)
	authCookie := http.Cookie{Name: authCookieName, Value: cookieVal}

	const newName = "New./-X"
	const oldName = "Old"

	profile := &userProfile{}
	profile.TOTPAuthData = make(map[int64]*totpAuthData)
	profile.TOTPAuthData[0] = &totpAuthData{Name: oldName}
	err = state.SaveUserProfile("username", profile)
	if err != nil {
		t.Fatal(err)
	}

	form := url.Values{}
	form.Add("username", "username")
	form.Add("index", "0")
	form.Add("name", newName)
	form.Add("action", "Update")

	req, err := http.NewRequest("POST", totpTokenManagementPath, strings.NewReader(form.Encode()))
	if err != nil {
		t.Fatal(err)
	}
	req.AddCookie(&authCookie)
	req.Header.Add("Content-Length", strconv.Itoa(len(form.Encode())))
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	val, err := checkRequestHandlerCode(req, state.totpTokenManagerHandler, http.StatusOK)
	if err != nil {
		t.Log(val)
		t.Fatal(err)
	}
	// Todo... check against the FS.
	profile, _, _, err = state.LoadUserProfile("username")
	if err != nil {
		t.Fatal(err)
	}
	if profile.TOTPAuthData[0].Name != newName {
		t.Fatal("update not successul")
	}
}
<commit_msg>fixing tests<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/Symantec/keymaster/lib/webapi/v0/proto"
	"github.com/pquerna/otp/totp"
)

func TestEncryptDecryptSuccess(t *testing.T) {
	state, passwdFile, err := setupValidRuntimeStateSigner()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(passwdFile.Name()) // clean up

	testSecret := "superSecret"
	cipherTexts, err := state.encryptWithPublicKeys([]byte(testSecret))
	if err != nil {
		t.Fatal(err)
	}
	plainTextBytes, err := state.decryptWithPublicKeys(cipherTexts)

	if err != nil {
		t.Fatal(err)
	}
	if string(plainTextBytes) != testSecret {
		t.Fatal("values do not match")
	}
}

func TestGenerateNewTOTPSuccess(t *testing.T) {
	state, passwdFile, err := setupValidRuntimeStateSigner()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(passwdFile.Name()) // clean up

	state.Config.Base.AllowedAuthBackendsForWebUI = append(state.Config.Base.AllowedAuthBackendsForWebUI, proto.AuthTypeU2F)

	state.signerPublicKeyToKeymasterKeys()

	dir, err := ioutil.TempDir("", "example")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up
	state.Config.Base.DataDirectory = dir
	err = initDB(state)
	if err != nil {
		t.Fatal(err)
	}
	state.HostIdentity = "testHost"

	// End of setup

	req, err := http.NewRequest("GET", totpGeneratNewPath, nil)
	if err != nil {
		t.Fatal(err)
		//return nil, err
	}
	cookieVal, err := state.setNewAuthCookie(nil, "username", AuthTypeU2F)
	if err != nil {
		t.Fatal(err)
	}
	authCookie := http.Cookie{Name: authCookieName, Value: cookieVal}
	req.AddCookie(&authCookie)

	tokenRR, err := checkRequestHandlerCode(req, state.GenerateNewTOTP, http.StatusOK)
	if err != nil {
		t.Fatal(err)
	}
	resultAccessToken := newTOTPPageTemplateData{}
	body := tokenRR.Result().Body
	err = json.NewDecoder(body).Decode(&resultAccessToken)
	if err != nil {
		t.Fatal(err)
	}
	t.Logf("totpDataToken='%+v'", resultAccessToken)

	// now we validate
	otpValue, err := totp.GenerateCode(resultAccessToken.TOTPSecret, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	data := url.Values{}
	data.Set("OTP", otpValue)
	t.Logf("url_Data='%+v'", data)
	validateReq, err := http.NewRequest("POST", totpValidateNewPath, bytes.NewBufferString(data.Encode()))
	if err != nil {
		t.Fatal(err)
		//return nil, err
	}
	validateReq.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	validateReq.AddCookie(&authCookie)
	_, err = checkRequestHandlerCode(validateReq, state.validateNewTOTP, http.StatusFound)
	if err != nil {
		t.Fatal(err)
	}
	//now check auth against it
	now := time.Now()
	otpValueInt, err := strconv.Atoi(otpValue)
	if err != nil {
		t.Fatal(err)
	}
	valid, err := state.validateUserTOTP("username", otpValueInt, now)
	if err != nil {
		t.Fatal(err)
	}
	if !valid {
		t.Fatal("should have been valid")
	}
	// now we retry with same value and should fail
	valid, err = state.validateUserTOTP("username", otpValueInt, now)
	if err != nil {
		t.Fatal(err)
	}
	if valid {
		t.Fatal("should NOT have been valid")
	}

}

func setupTestStateWithTOTPSecret(t *testing.T, state *RuntimeState, cookieAuth int) (*http.Cookie, string, error) {

	authUser := "username"
	key, err := totp.Generate(totp.GenerateOpts{
		Issuer:      "keymaster-totp", //TODO: get the actual name
		AccountName: authUser,
	})
	if err != nil {
		t.Fatal(err)
	}
	profile, _, _, err := state.LoadUserProfile(authUser)
	if err != nil {
		t.Fatal(err)
	}
	encryptedKeys, err := state.encryptWithPublicKeys([]byte(key.Secret()))
	if err != nil {
		t.Fatal(err)
	}
	newTOTPAuthData := totpAuthData{
		CreatedAt:       time.Now(),
		EncryptedSecret: encryptedKeys,
		//ValidatorAddr:   r.RemoteAddr,
		Enabled: true,
	}
	newIndex := newTOTPAuthData.CreatedAt.Unix()
	profile.TOTPAuthData[newIndex] = &newTOTPAuthData

	err = state.SaveUserProfile(authUser, profile)
	if err != nil {
		t.Fatal(err)
	}
	cookieVal, err := state.setNewAuthCookie(nil, "username", cookieAuth)
	if err != nil {
		t.Fatal(err)
	}
	authCookie := http.Cookie{Name: authCookieName, Value: cookieVal}
	return &authCookie, key.Secret(), nil
}

func TestVerifyTOTPHandlerSuccess(t *testing.T) {
	state, passwdFile, err := setupValidRuntimeStateSigner()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(passwdFile.Name()) // clean up

	state.Config.Base.AllowedAuthBackendsForWebUI = append(state.Config.Base.AllowedAuthBackendsForWebUI, proto.AuthTypeU2F)

	state.signerPublicKeyToKeymasterKeys()

	dir, err := ioutil.TempDir("", "example-1")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up
	state.Config.Base.DataDirectory = dir
	err = initDB(state)
	if err != nil {
		t.Fatal(err)
	}

	// End of setup
	authCookie, totpSecret, err := setupTestStateWithTOTPSecret(t, state, AuthTypeU2F)
	if err != nil {
		t.Fatal(err)
	}
	otpValue, err := totp.GenerateCode(totpSecret, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	data := url.Values{}
	data.Set("OTP", otpValue)
	t.Logf("i testverify url_Data='%+v'", data)
	verifyReq, err := http.NewRequest("POST", totpValidateNewPath, bytes.NewBufferString(data.Encode()))
	if err != nil {
		t.Fatal(err)
	}
	verifyReq.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	verifyReq.AddCookie(authCookie)
	_, err = checkRequestHandlerCode(verifyReq, state.verifyTOTPHandler, http.StatusFound)
	if err != nil {
		t.Fatal(err)
	}
}

func TestAuthTOTPHandlerSuccess(t *testing.T) {
	state, passwdFile, err := setupValidRuntimeStateSigner()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(passwdFile.Name()) // clean up

	state.Config.Base.AllowedAuthBackendsForWebUI = append(state.Config.Base.AllowedAuthBackendsForWebUI, proto.AuthTypeTOTP)

	state.signerPublicKeyToKeymasterKeys()

	dir, err := ioutil.TempDir("", "example")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up
	state.Config.Base.DataDirectory = dir
	err = initDB(state)
	if err != nil {
		t.Fatal(err)
	}
	// End of setup
	authCookie, totpSecret, err := setupTestStateWithTOTPSecret(t, state, AuthTypePassword)
	if err != nil {
		t.Fatal(err)
	}

	otpValue, err := totp.GenerateCode(totpSecret, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	data := url.Values{}
	data.Set("OTP", otpValue)
	t.Logf("i testverify url_Data='%+v'", data)
	verifyReq, err := http.NewRequest("POST", totpValidateNewPath, bytes.NewBufferString(data.Encode()))
	if err != nil {
		t.Fatal(err)
	}
	verifyReq.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	verifyReq.AddCookie(authCookie)
	_, err = checkRequestHandlerCode(verifyReq, state.TOTPAuthHandler, http.StatusOK)
	if err != nil {
		t.Fatal(err)
	}
}

func TestTOTPTokenManagerHandlerUpdateSuccess(t *testing.T) {
	state, passwdFile, err := setupValidRuntimeStateSigner()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(passwdFile.Name()) // clean up

	state.Config.Base.AllowedAuthBackendsForWebUI = append(state.Config.Base.AllowedAuthBackendsForWebUI, proto.AuthTypePassword)

	state.signerPublicKeyToKeymasterKeys()

	dir, err := ioutil.TempDir("", "example")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir) // clean up
	state.Config.Base.DataDirectory = dir
	err = initDB(state)
	if err != nil {
		t.Fatal(err)
	}
	//
	cookieVal, err := state.setNewAuthCookie(nil, "username", AuthTypeAny)
	if err != nil {
		t.Fatal(err)
	}
	//cookieReq.AddCookie(&authCookie)
	authCookie := http.Cookie{Name: authCookieName, Value: cookieVal}

	const newName = "New./-X"
	const oldName = "Old"

	profile := &userProfile{}
	profile.TOTPAuthData = make(map[int64]*totpAuthData)
	profile.TOTPAuthData[0] = &totpAuthData{Name: oldName}
	err = state.SaveUserProfile("username", profile)
	if err != nil {
		t.Fatal(err)
	}

	form := url.Values{}
	form.Add("username", "username")
	form.Add("index", "0")
	form.Add("name", newName)
	form.Add("action", "Update")

	req, err := http.NewRequest("POST", totpTokenManagementPath, strings.NewReader(form.Encode()))
	if err != nil {
		t.Fatal(err)
	}
	req.AddCookie(&authCookie)
	req.Header.Add("Content-Length", strconv.Itoa(len(form.Encode())))
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	val, err := checkRequestHandlerCode(req, state.totpTokenManagerHandler, http.StatusOK)
	if err != nil {
		t.Log(val)
		t.Fatal(err)
	}
	// Todo... check against the FS.
	profile, _, _, err = state.LoadUserProfile("username")
	if err != nil {
		t.Fatal(err)
	}
	if profile.TOTPAuthData[0].Name != newName {
		t.Fatal("update not successul")
	}
}
<|endoftext|>


<commit_before>package oci

import (
	"context"
	"path/filepath"
	"strings"

	"github.com/containerd/containerd/containers"
	"github.com/containerd/containerd/oci"
	specs "github.com/opencontainers/runtime-spec/specs-go"
)

func withRemovedMount(destination string) oci.SpecOpts {
	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
		newMounts := []specs.Mount{}
		for _, o := range s.Mounts {
			if o.Destination != destination {
				newMounts = append(newMounts, o)
			}
		}
		s.Mounts = newMounts

		return nil
	}
}

func withROBind(src, dest string) oci.SpecOpts {
	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
		s.Mounts = append(s.Mounts, specs.Mount{
			Destination: dest,
			Type:        "bind",
			Source:      src,
			Options:     []string{"nosuid", "noexec", "nodev", "rbind", "ro"},
		})
		return nil
	}
}

func withCGroup() oci.SpecOpts {
	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
		s.Mounts = append(s.Mounts, specs.Mount{
			Destination: "/sys/fs/cgroup",
			Type:        "cgroup",
			Source:      "cgroup",
			Options:     []string{"ro", "nosuid", "noexec", "nodev"},
		})
		return nil
	}

}

func hasPrefix(p, prefixDir string) bool {
	prefixDir = filepath.Clean(prefixDir)
	if filepath.Base(prefixDir) == string(filepath.Separator) {
		return true
	}
	p = filepath.Clean(p)
	return p == prefixDir || strings.HasPrefix(p, prefixDir+string(filepath.Separator))
}

func removeMountsWithPrefix(mounts []specs.Mount, prefixDir string) []specs.Mount {
	var ret []specs.Mount
	for _, m := range mounts {
		if !hasPrefix(m.Destination, prefixDir) {
			ret = append(ret, m)
		}
	}
	return ret
}

func withBoundProc() oci.SpecOpts {
	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
		s.Mounts = removeMountsWithPrefix(s.Mounts, "/proc")
		procMount := specs.Mount{
			Destination: "/proc",
			Type:        "bind",
			Source:      "/proc",
			// NOTE: "rbind"+"ro" does not make /proc read-only recursively.
			// So we keep maskedPath and readonlyPaths (although not mandatory for rootless mode)
			Options: []string{"rbind"},
		}
		s.Mounts = append([]specs.Mount{procMount}, s.Mounts...)

		var maskedPaths []string
		for _, s := range s.Linux.MaskedPaths {
			if !hasPrefix(s, "/proc") {
				maskedPaths = append(maskedPaths, s)
			}
		}
		s.Linux.MaskedPaths = maskedPaths

		var readonlyPaths []string
		for _, s := range s.Linux.ReadonlyPaths {
			if !hasPrefix(s, "/proc") {
				readonlyPaths = append(readonlyPaths, s)
			}
		}
		s.Linux.ReadonlyPaths = readonlyPaths

		return nil
	}
}

func dedupMounts(mnts []specs.Mount) []specs.Mount {
	ret := make([]specs.Mount, 0, len(mnts))
	visited := make(map[string]int)
	for i, mnt := range mnts {
		if j, ok := visited[mnt.Destination]; ok {
			ret[j] = mnt
		} else {
			visited[mnt.Destination] = i
			ret = append(ret, mnt)
		}
	}
	return ret
}
<commit_msg>Fix out-of-bounds panic in dedupMounts<commit_after>package oci

import (
	"context"
	"path/filepath"
	"strings"

	"github.com/containerd/containerd/containers"
	"github.com/containerd/containerd/oci"
	specs "github.com/opencontainers/runtime-spec/specs-go"
)

func withRemovedMount(destination string) oci.SpecOpts {
	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
		newMounts := []specs.Mount{}
		for _, o := range s.Mounts {
			if o.Destination != destination {
				newMounts = append(newMounts, o)
			}
		}
		s.Mounts = newMounts

		return nil
	}
}

func withROBind(src, dest string) oci.SpecOpts {
	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
		s.Mounts = append(s.Mounts, specs.Mount{
			Destination: dest,
			Type:        "bind",
			Source:      src,
			Options:     []string{"nosuid", "noexec", "nodev", "rbind", "ro"},
		})
		return nil
	}
}

func withCGroup() oci.SpecOpts {
	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
		s.Mounts = append(s.Mounts, specs.Mount{
			Destination: "/sys/fs/cgroup",
			Type:        "cgroup",
			Source:      "cgroup",
			Options:     []string{"ro", "nosuid", "noexec", "nodev"},
		})
		return nil
	}

}

func hasPrefix(p, prefixDir string) bool {
	prefixDir = filepath.Clean(prefixDir)
	if filepath.Base(prefixDir) == string(filepath.Separator) {
		return true
	}
	p = filepath.Clean(p)
	return p == prefixDir || strings.HasPrefix(p, prefixDir+string(filepath.Separator))
}

func removeMountsWithPrefix(mounts []specs.Mount, prefixDir string) []specs.Mount {
	var ret []specs.Mount
	for _, m := range mounts {
		if !hasPrefix(m.Destination, prefixDir) {
			ret = append(ret, m)
		}
	}
	return ret
}

func withBoundProc() oci.SpecOpts {
	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
		s.Mounts = removeMountsWithPrefix(s.Mounts, "/proc")
		procMount := specs.Mount{
			Destination: "/proc",
			Type:        "bind",
			Source:      "/proc",
			// NOTE: "rbind"+"ro" does not make /proc read-only recursively.
			// So we keep maskedPath and readonlyPaths (although not mandatory for rootless mode)
			Options: []string{"rbind"},
		}
		s.Mounts = append([]specs.Mount{procMount}, s.Mounts...)

		var maskedPaths []string
		for _, s := range s.Linux.MaskedPaths {
			if !hasPrefix(s, "/proc") {
				maskedPaths = append(maskedPaths, s)
			}
		}
		s.Linux.MaskedPaths = maskedPaths

		var readonlyPaths []string
		for _, s := range s.Linux.ReadonlyPaths {
			if !hasPrefix(s, "/proc") {
				readonlyPaths = append(readonlyPaths, s)
			}
		}
		s.Linux.ReadonlyPaths = readonlyPaths

		return nil
	}
}

func dedupMounts(mnts []specs.Mount) []specs.Mount {
	ret := make([]specs.Mount, 0, len(mnts))
	visited := make(map[string]int)
	for _, mnt := range mnts {
		if j, ok := visited[mnt.Destination]; ok {
			ret[j] = mnt
		} else {
			visited[mnt.Destination] = len(ret)
			ret = append(ret, mnt)
		}
	}
	return ret
}
<|endoftext|>


<commit_before>package expprod

import (
	"encoding/json"
	"net/url"
	"path"
	"sync"

	"github.com/spf13/cobra"
	"github.com/srinandan/apigeecli/cmd/shared"
)

var Cmd = &cobra.Command{
	Use:   "export",
	Short: "Export API products to a file",
	Long:  "Export API products to a file",
	RunE: func(cmd *cobra.Command, args []string) (err error) {

		const exportFileName = "products.json"
		err = exportProducts()
		if err != nil {
			return
		}
		return shared.WriteJSONArrayToFile(exportFileName)
	},
}

type apiProducts struct {
	APIProducts []apiProduct `json:"apiProduct,omitempty"`
}

type apiProduct struct {
	Name string `json:"name,omitempty"`
}

var conn int

func init() {

	Cmd.Flags().IntVarP(&conn, "conn", "c",
		4, "Number of connections")

}

//batch created a batch of products to query
func batch(apiProductNames []apiProduct, entityType string, pwg *sync.WaitGroup, mu *sync.Mutex) {

	defer pwg.Done()
	//batch workgroup
	var bwg sync.WaitGroup

	bwg.Add(len(apiProductNames))

	for _, apiProductName := range apiProductNames {
		go shared.GetAsyncEntity(url.PathEscape(apiProductName.Name), entityType, &bwg, mu)
	}
	bwg.Wait()
}

func exportProducts() error {

	//parent workgroup
	var pwg sync.WaitGroup
	var mu sync.Mutex
	const entityType = "apiproducts"

	u, _ := url.Parse(shared.BaseURL)
	u.Path = path.Join(u.Path, shared.RootArgs.Org, entityType)
	//don't print to sysout
	respBody, err := shared.HttpClient(false, u.String())
	if err != nil {
		return err
	}

	var products = apiProducts{}
	err = json.Unmarshal(respBody, &products)
	if err != nil {
		return err
	}

	numProd := len(products.APIProducts)
	shared.Info.Printf("Found %d products in the org\n", numProd)
	shared.Info.Printf("Exporting products with %d connections\n", conn)

	numOfLoops, remaining := numProd/conn, numProd%conn

	//ensure connections aren't greater then products
	if conn > numProd {
		conn = numProd
	}

	start := 0

	for i, end := 0, 0; i < numOfLoops; i++ {
		pwg.Add(1)
		end = (i * conn) + conn
		shared.Info.Printf("Exporting batch %d of products\n", (i + 1))
		go batch(products.APIProducts[start:end], entityType, &pwg, &mu)
		start = end
		pwg.Wait()
	}

	if remaining > 0 {
		pwg.Add(1)
		shared.Info.Printf("Exporting remaining %d products\n", remaining)
		go batch(products.APIProducts[start:numProd], entityType, &pwg, &mu)
		pwg.Wait()
	}

	return nil
}
<commit_msg>update var names<commit_after>package expprod

import (
	"encoding/json"
	"net/url"
	"path"
	"sync"

	"github.com/spf13/cobra"
	"github.com/srinandan/apigeecli/cmd/shared"
)

var Cmd = &cobra.Command{
	Use:   "export",
	Short: "Export API products to a file",
	Long:  "Export API products to a file",
	RunE: func(cmd *cobra.Command, args []string) (err error) {

		const exportFileName = "products.json"
		err = exportProducts()
		if err != nil {
			return
		}
		return shared.WriteJSONArrayToFile(exportFileName)
	},
}

type apiProducts struct {
	APIProducts []apiProduct `json:"apiProduct,omitempty"`
}

type apiProduct struct {
	Name string `json:"name,omitempty"`
}

var conn int

func init() {

	Cmd.Flags().IntVarP(&conn, "conn", "c",
		4, "Number of connections")

}

//batch created a batch of products to query
func batch(entities []apiProduct, entityType string, pwg *sync.WaitGroup, mu *sync.Mutex) {

	defer pwg.Done()
	//batch workgroup
	var bwg sync.WaitGroup

	bwg.Add(len(entities))

	for _, entity := range entities {
		go shared.GetAsyncEntity(url.PathEscape(entity.Name), entityType, &bwg, mu)
	}
	bwg.Wait()
}

func exportProducts() error {

	//parent workgroup
	var pwg sync.WaitGroup
	var mu sync.Mutex
	const entityType = "apiproducts"

	u, _ := url.Parse(shared.BaseURL)
	u.Path = path.Join(u.Path, shared.RootArgs.Org, entityType)
	//don't print to sysout
	respBody, err := shared.HttpClient(false, u.String())
	if err != nil {
		return err
	}

	var products = apiProducts{}
	err = json.Unmarshal(respBody, &products)
	if err != nil {
		return err
	}

	numProd := len(products.APIProducts)
	shared.Info.Printf("Found %d products in the org\n", numProd)
	shared.Info.Printf("Exporting products with %d connections\n", conn)

	numOfLoops, remaining := numProd/conn, numProd%conn

	//ensure connections aren't greater than products
	if conn > numProd {
		conn = numProd
	}

	start := 0

	for i, end := 0, 0; i < numOfLoops; i++ {
		pwg.Add(1)
		end = (i * conn) + conn
		shared.Info.Printf("Exporting batch %d of products\n", (i + 1))
		go batch(products.APIProducts[start:end], entityType, &pwg, &mu)
		start = end
		pwg.Wait()
	}

	if remaining > 0 {
		pwg.Add(1)
		shared.Info.Printf("Exporting remaining %d products\n", remaining)
		go batch(products.APIProducts[start:numProd], entityType, &pwg, &mu)
		pwg.Wait()
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"database/sql"
	"encoding/gob"
	"errors"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"os"
	"path/filepath"
	"strings"
)

const userProfilePrefix = "profile_"
const userProfileSuffix = ".gob"
const profileDBFilename = "userProfiles.sqlite3"

func initDB(state *RuntimeState) error {
	log.Printf("storage=%s", state.Config.ProfileStorage.StorageUrl)
	storageURL := state.Config.ProfileStorage.StorageUrl
	if storageURL == "" {
		storageURL = "sqlite:"
	}
	splitString := strings.SplitN(storageURL, ":", 2)
	if len(splitString) < 1 {
		log.Printf("invalid string")
		err := errors.New("Bad storage url string")
		return err
	}
	switch splitString[0] {
	case "sqlite":
		log.Printf("doing sqlite")
		return initDBSQlite(state)
	case "postgresql":
		log.Printf("doing postgres")
		return initDBPostgres(state)
	default:
		log.Printf("invalid storage url string")
		err := errors.New("Bad storage url string")
		return err
	}

	err := errors.New("invalid state")
	return err

}

func initDBPostgres(state *RuntimeState) (err error) {
	state.dbType = "postgres"
	state.db, err = sql.Open("postgres", state.Config.ProfileStorage.StorageUrl)
	if err != nil {
		return err
	}
	/// This should be changed to take care of DB schema
	if true {
		sqlStmt := `create table if not exists user_profile (id serial not null primary key, username text unique, profile_data bytea);`
		_, err = state.db.Exec(sqlStmt)
		if err != nil {
			log.Printf("%q: %s\n", err, sqlStmt)
			return err
		}
	}

	return nil
}

// This call initializes the database if it does not exist.
// TODO: update to handle multiple db types AND to perform auto-updates of the db.
func initDBSQlite(state *RuntimeState) (err error) {
	state.dbType = "sqlite"
	dbFilename := filepath.Join(state.Config.Base.DataDirectory, profileDBFilename)
	if _, err := os.Stat(dbFilename); os.IsNotExist(err) {
		//CREATE NEW DB
		state.db, err = sql.Open("sqlite3", dbFilename)
		if err != nil {
			return err
		}
		log.Printf("post DB open")
		// create profile table
		sqlStmt := `create table user_profile (id integer not null primary key, username text unique, profile_data blob);`
		_, err = state.db.Exec(sqlStmt)
		if err != nil {
			log.Printf("%q: %s\n", err, sqlStmt)
			return err
		}

		return nil
	}

	// try open the DB
	if state.db == nil {

		state.db, err = sql.Open("sqlite3", dbFilename)
		if err != nil {
			return err
		}
		//defer state.db.Close()
	}
	return nil
}

var loadUserProfileStmt = map[string]string{
	"sqlite":   "select profile_data from user_profile where username = ?",
	"postgres": "select profile_data from user_profile where username = $1",
}

/// Adding api to be load/save per user

// Notice: each operation load/save should be atomic. For inital version we
// are using a RWMutex to al least serialize writes and allow for some
// read parallelism. Once SQL is implemented this RWMutex should be removed

// If there a valid user profile returns: profile, true nil
// If there is NO user profile returns default_object, false, nil
// Any other case: nil, false, error
func (state *RuntimeState) LoadUserProfile(username string) (profile *userProfile, ok bool, err error) {
	var defaultProfile userProfile
	defaultProfile.U2fAuthData = make(map[int64]*u2fAuthData)
	//load from DB
	stmtText := loadUserProfileStmt[state.dbType]
	stmt, err := state.db.Prepare(stmtText)
	if err != nil {
		log.Print("Error Preparing statement")
		log.Fatal(err)
	}

	defer stmt.Close()
	var profileBytes []byte
	err = stmt.QueryRow(username).Scan(&profileBytes)
	if err != nil {
		if err.Error() == "sql: no rows in result set" {
			log.Printf("err='%s'", err)
			return &defaultProfile, false, nil
		} else {
			log.Printf("Problem with db ='%s'", err)
			return nil, false, err
		}

	}
	//log.Printf("bytes len=%d", len(profileBytes))
	//gobReader := bytes.NewReader(fileBytes)
	gobReader := bytes.NewReader(profileBytes)
	decoder := gob.NewDecoder(gobReader)
	err = decoder.Decode(&defaultProfile)
	if err != nil {
		return nil, false, err
	}
	//log.Printf("loaded profile=%+v", defaultProfile)
	return &defaultProfile, true, nil
}

var saveUserProfileStmt = map[string]string{
	"sqlite":   "insert or replace into user_profile(username, profile_data) values(?, ?)",
	"postgres": "insert into user_profile(username, profile_data) values ($1,$2) on CONFLICT(username) DO UPDATE set  profile_data = excluded.profile_data",
}

func (state *RuntimeState) SaveUserProfile(username string, profile *userProfile) error {
	var gobBuffer bytes.Buffer

	encoder := gob.NewEncoder(&gobBuffer)
	if err := encoder.Encode(profile); err != nil {
		return err
	}

	//insert into DB
	tx, err := state.db.Begin()
	if err != nil {
		return err
	}
	stmtText := saveUserProfileStmt[state.dbType]
	stmt, err := tx.Prepare(stmtText)
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(username, gobBuffer.Bytes())
	if err != nil {
		return err
	}
	err = tx.Commit()
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>only print storage url in debug mode (dont leak it via logs)<commit_after>package main

import (
	"bytes"
	"database/sql"
	"encoding/gob"
	"errors"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"os"
	"path/filepath"
	"strings"
)

const userProfilePrefix = "profile_"
const userProfileSuffix = ".gob"
const profileDBFilename = "userProfiles.sqlite3"

func initDB(state *RuntimeState) error {
	if *debug {
		log.Printf("storage=%s", state.Config.ProfileStorage.StorageUrl)
	}
	storageURL := state.Config.ProfileStorage.StorageUrl
	if storageURL == "" {
		storageURL = "sqlite:"
	}
	splitString := strings.SplitN(storageURL, ":", 2)
	if len(splitString) < 1 {
		log.Printf("invalid string")
		err := errors.New("Bad storage url string")
		return err
	}
	switch splitString[0] {
	case "sqlite":
		log.Printf("doing sqlite")
		return initDBSQlite(state)
	case "postgresql":
		log.Printf("doing postgres")
		return initDBPostgres(state)
	default:
		log.Printf("invalid storage url string")
		err := errors.New("Bad storage url string")
		return err
	}

	err := errors.New("invalid state")
	return err

}

func initDBPostgres(state *RuntimeState) (err error) {
	state.dbType = "postgres"
	state.db, err = sql.Open("postgres", state.Config.ProfileStorage.StorageUrl)
	if err != nil {
		return err
	}
	/// This should be changed to take care of DB schema
	if true {
		sqlStmt := `create table if not exists user_profile (id serial not null primary key, username text unique, profile_data bytea);`
		_, err = state.db.Exec(sqlStmt)
		if err != nil {
			log.Printf("%q: %s\n", err, sqlStmt)
			return err
		}
	}

	return nil
}

// This call initializes the database if it does not exist.
// TODO: update to handle multiple db types AND to perform auto-updates of the db.
func initDBSQlite(state *RuntimeState) (err error) {
	state.dbType = "sqlite"
	dbFilename := filepath.Join(state.Config.Base.DataDirectory, profileDBFilename)
	if _, err := os.Stat(dbFilename); os.IsNotExist(err) {
		//CREATE NEW DB
		state.db, err = sql.Open("sqlite3", dbFilename)
		if err != nil {
			return err
		}
		log.Printf("post DB open")
		// create profile table
		sqlStmt := `create table user_profile (id integer not null primary key, username text unique, profile_data blob);`
		_, err = state.db.Exec(sqlStmt)
		if err != nil {
			log.Printf("%q: %s\n", err, sqlStmt)
			return err
		}

		return nil
	}

	// try open the DB
	if state.db == nil {

		state.db, err = sql.Open("sqlite3", dbFilename)
		if err != nil {
			return err
		}
		//defer state.db.Close()
	}
	return nil
}

var loadUserProfileStmt = map[string]string{
	"sqlite":   "select profile_data from user_profile where username = ?",
	"postgres": "select profile_data from user_profile where username = $1",
}

/// Adding api to be load/save per user

// Notice: each operation load/save should be atomic. For inital version we
// are using a RWMutex to al least serialize writes and allow for some
// read parallelism. Once SQL is implemented this RWMutex should be removed

// If there a valid user profile returns: profile, true nil
// If there is NO user profile returns default_object, false, nil
// Any other case: nil, false, error
func (state *RuntimeState) LoadUserProfile(username string) (profile *userProfile, ok bool, err error) {
	var defaultProfile userProfile
	defaultProfile.U2fAuthData = make(map[int64]*u2fAuthData)
	//load from DB
	stmtText := loadUserProfileStmt[state.dbType]
	stmt, err := state.db.Prepare(stmtText)
	if err != nil {
		log.Print("Error Preparing statement")
		log.Fatal(err)
	}

	defer stmt.Close()
	var profileBytes []byte
	err = stmt.QueryRow(username).Scan(&profileBytes)
	if err != nil {
		if err.Error() == "sql: no rows in result set" {
			log.Printf("err='%s'", err)
			return &defaultProfile, false, nil
		} else {
			log.Printf("Problem with db ='%s'", err)
			return nil, false, err
		}

	}
	//log.Printf("bytes len=%d", len(profileBytes))
	//gobReader := bytes.NewReader(fileBytes)
	gobReader := bytes.NewReader(profileBytes)
	decoder := gob.NewDecoder(gobReader)
	err = decoder.Decode(&defaultProfile)
	if err != nil {
		return nil, false, err
	}
	//log.Printf("loaded profile=%+v", defaultProfile)
	return &defaultProfile, true, nil
}

var saveUserProfileStmt = map[string]string{
	"sqlite":   "insert or replace into user_profile(username, profile_data) values(?, ?)",
	"postgres": "insert into user_profile(username, profile_data) values ($1,$2) on CONFLICT(username) DO UPDATE set  profile_data = excluded.profile_data",
}

func (state *RuntimeState) SaveUserProfile(username string, profile *userProfile) error {
	var gobBuffer bytes.Buffer

	encoder := gob.NewEncoder(&gobBuffer)
	if err := encoder.Encode(profile); err != nil {
		return err
	}

	//insert into DB
	tx, err := state.db.Begin()
	if err != nil {
		return err
	}
	stmtText := saveUserProfileStmt[state.dbType]
	stmt, err := tx.Prepare(stmtText)
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(username, gobBuffer.Bytes())
	if err != nil {
		return err
	}
	err = tx.Commit()
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package spi

import (
	"encoding/base64"
	"fmt"
	"log"
)

//Common Variables -------------------------------------------------------------

const XPS_HTTPS = API_HTTPS + "/Experiments"

//API calls --------------------------------------------------------------------

func CreateExperiment(expId, owner, topdl string) (
	*CreateExperimentResponse, error) {

	e := CreateExperimentEnvelope{}
	e.Body.CreateExperiment.EID = expId
	e.Body.CreateExperiment.Owner = owner
	e.Body.CreateExperiment.Aspects = append(e.Body.CreateExperiment.Aspects, ExperimentAspect{
		Data: base64.StdEncoding.EncodeToString([]byte(topdl)),
		Type: "layout",
	})
	e.Body.CreateExperiment.Profile = append(e.Body.CreateExperiment.Profile, DescriptionAttr{
		"Description", "This is not an experiment"})

	var responseEnvelope CreateExperimentResponseEnvelope

	rsp, _, err := spiCall(XPS_HTTPS+"/createExperiment", e, &responseEnvelope)
	if err != nil {
		log.Println(err)
		return nil, err
	}

	if rsp.StatusCode != 200 {
		return nil, fmt.Errorf("Server did not accept the createExperiment call - %d",
			rsp.StatusCode)
	}

	response := &responseEnvelope.Body.CreateExperimentResponse

	return response, nil
}

func RemoveExperiment(expId string) (*RemoveExperimentResponse, error) {

	e := RemoveExperimentEnvelope{}
	e.Body.RemoveExperiment.EID = expId

	var responseEnvelope RemoveExperimentResponseEnvelope

	rsp, _, err := spiCall(XPS_HTTPS+"/removeExperiment", e, &responseEnvelope)
	if err != nil {
		log.Println(err)
		return nil, err
	}

	if rsp.StatusCode != 200 {
		return nil, fmt.Errorf("Server did not accept the removeExperiment call - %d",
			rsp.StatusCode)
	}

	response := &responseEnvelope.Body.RemoveExperimentResponse

	return response, nil
}
<commit_msg>lower case attribute names<commit_after>package spi

import (
	"encoding/base64"
	"fmt"
	"log"
)

//Common Variables -------------------------------------------------------------

const XPS_HTTPS = API_HTTPS + "/Experiments"

//API calls --------------------------------------------------------------------

func CreateExperiment(expId, owner, topdl string) (
	*CreateExperimentResponse, error) {

	e := CreateExperimentEnvelope{}
	e.Body.CreateExperiment.EID = expId
	e.Body.CreateExperiment.Owner = owner
	e.Body.CreateExperiment.Aspects = append(e.Body.CreateExperiment.Aspects, ExperimentAspect{
		Data: base64.StdEncoding.EncodeToString([]byte(topdl)),
		Type: "layout",
	})
	e.Body.CreateExperiment.Profile = append(e.Body.CreateExperiment.Profile, DescriptionAttr{
		"description", "This is not an experiment"})

	var responseEnvelope CreateExperimentResponseEnvelope

	rsp, _, err := spiCall(XPS_HTTPS+"/createExperiment", e, &responseEnvelope)
	if err != nil {
		log.Println(err)
		return nil, err
	}

	if rsp.StatusCode != 200 {
		return nil, fmt.Errorf("Server did not accept the createExperiment call - %d",
			rsp.StatusCode)
	}

	response := &responseEnvelope.Body.CreateExperimentResponse

	return response, nil
}

func RemoveExperiment(expId string) (*RemoveExperimentResponse, error) {

	e := RemoveExperimentEnvelope{}
	e.Body.RemoveExperiment.EID = expId

	var responseEnvelope RemoveExperimentResponseEnvelope

	rsp, _, err := spiCall(XPS_HTTPS+"/removeExperiment", e, &responseEnvelope)
	if err != nil {
		log.Println(err)
		return nil, err
	}

	if rsp.StatusCode != 200 {
		return nil, fmt.Errorf("Server did not accept the removeExperiment call - %d",
			rsp.StatusCode)
	}

	response := &responseEnvelope.Body.RemoveExperimentResponse

	return response, nil
}
<|endoftext|>


<commit_before>package v1

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/dpb587/bosh-compiled-releases/cli/repository"
	"github.com/dpb587/bosh-compiled-releases/cli/server/v1/model"
	"github.com/sirupsen/logrus"
)

type resolve struct {
	repo   repository.ReadableRepository
	logger logrus.FieldLogger
}

var _ http.Handler = resolve{}

func NewResolve(logger logrus.FieldLogger, repo repository.ReadableRepository) http.Handler {
	return resolve{
		repo:   repo,
		logger: logger,
	}
}

func (h resolve) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	logger := h.logger.WithFields(logrus.Fields{
		"request.remote_addr": r.RemoteAddr,
		"request.method":      r.Method,
		"request.uri":         r.RequestURI,
	})

	if r.Method != "GET" {
		w.Header().Add("Allow", "GET")
		http.Error(w, fmt.Sprintf("405 Method Not Allowed"), http.StatusMethodNotAllowed)
		logger.WithField("response.status", http.StatusMethodNotAllowed).Warn("method not allowed")

		return
	}

	rBytes, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, fmt.Sprintf("400 Bad Request\n%s", err), http.StatusBadRequest)
		logger.WithField("response.status", http.StatusBadRequest).Warn("bad request body: %s", err)

		return
	}

	var data = model.ResolveRequest{}

	err = json.Unmarshal(rBytes, &data)
	if err != nil {
		http.Error(w, fmt.Sprintf("400 Bad Request\n%s", err), http.StatusBadRequest)
		logger.WithField("response.status", http.StatusBadRequest).Warnf("bad request body data: %s", err)

		return
	}

	logger = logger.WithFields(logrus.Fields{
		"handler.resolve.release_name":     data.Name,
		"handler.resolve.release_version":  data.Version,
		"handler.resolve.source_digest":    data.Sha1,
		"handler.resolve.stemcell_os":      data.Stemcell.OS,
		"handler.resolve.stemcell_version": data.Stemcell.Version,
	})

	compiledRelease, err := h.repo.Find(
		data.Name,
		data.Version,
		repository.SourceRelease{
			Digest: data.Sha1,
		},
		repository.CompiledReleaseStemcell{
			OS:      data.Stemcell.OS,
			Version: data.Stemcell.Version,
		},
	)
	if err != nil {
		http.Error(w, fmt.Sprintf("500 Internal Server Error"), http.StatusInternalServerError)
		logger.WithField("response.status", http.StatusInternalServerError).Errorf("finding compiled releases: %s", err)

		return
	} else if compiledRelease == nil {
		http.Error(w, fmt.Sprintf("404 Not Found"), http.StatusNotFound)
		logger.WithField("response.status", http.StatusNotFound).Info("missing compiled release")

		return
	}

	wBytes, err := json.Marshal(model.ResolveResponse{
		CompiledRelease: model.ResolveResponseCompiledRelease{
			URL:  compiledRelease.URL,
			Sha1: compiledRelease.Digest,
		},
	})
	if err != nil {
		http.Error(w, fmt.Sprintf("500 Internal Server Error"), http.StatusInternalServerError)
		logger.WithField("response.status", http.StatusInternalServerError).Error(err)

		return
	}

	w.WriteHeader(http.StatusOK)
	w.Write(wBytes)

	logger.WithField("response.status", http.StatusOK).Info("found compiled release")
}
<commit_msg>wording<commit_after>package v1

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/dpb587/bosh-compiled-releases/cli/repository"
	"github.com/dpb587/bosh-compiled-releases/cli/server/v1/model"
	"github.com/sirupsen/logrus"
)

type resolve struct {
	repo   repository.ReadableRepository
	logger logrus.FieldLogger
}

var _ http.Handler = resolve{}

func NewResolve(logger logrus.FieldLogger, repo repository.ReadableRepository) http.Handler {
	return resolve{
		repo:   repo,
		logger: logger,
	}
}

func (h resolve) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	logger := h.logger.WithFields(logrus.Fields{
		"request.remote_addr": r.RemoteAddr,
		"request.method":      r.Method,
		"request.uri":         r.RequestURI,
	})

	if r.Method != "GET" {
		w.Header().Add("Allow", "GET")
		http.Error(w, fmt.Sprintf("405 Method Not Allowed"), http.StatusMethodNotAllowed)
		logger.WithField("response.status", http.StatusMethodNotAllowed).Warn("method not allowed")

		return
	}

	rBytes, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, fmt.Sprintf("400 Bad Request\n%s", err), http.StatusBadRequest)
		logger.WithField("response.status", http.StatusBadRequest).Warn("bad request body: %s", err)

		return
	}

	var data = model.ResolveRequest{}

	err = json.Unmarshal(rBytes, &data)
	if err != nil {
		http.Error(w, fmt.Sprintf("400 Bad Request\n%s", err), http.StatusBadRequest)
		logger.WithField("response.status", http.StatusBadRequest).Warnf("bad request body data: %s", err)

		return
	}

	logger = logger.WithFields(logrus.Fields{
		"handler.resolve.release_name":     data.Name,
		"handler.resolve.release_version":  data.Version,
		"handler.resolve.source_digest":    data.Sha1,
		"handler.resolve.stemcell_os":      data.Stemcell.OS,
		"handler.resolve.stemcell_version": data.Stemcell.Version,
	})

	compiledRelease, err := h.repo.Find(
		data.Name,
		data.Version,
		repository.SourceRelease{
			Digest: data.Sha1,
		},
		repository.CompiledReleaseStemcell{
			OS:      data.Stemcell.OS,
			Version: data.Stemcell.Version,
		},
	)
	if err != nil {
		http.Error(w, fmt.Sprintf("500 Internal Server Error"), http.StatusInternalServerError)
		logger.WithField("response.status", http.StatusInternalServerError).Errorf("finding compiled release: %s", err)

		return
	} else if compiledRelease == nil {
		http.Error(w, fmt.Sprintf("404 Not Found"), http.StatusNotFound)
		logger.WithField("response.status", http.StatusNotFound).Info("missing compiled release")

		return
	}

	wBytes, err := json.Marshal(model.ResolveResponse{
		CompiledRelease: model.ResolveResponseCompiledRelease{
			URL:  compiledRelease.URL,
			Sha1: compiledRelease.Digest,
		},
	})
	if err != nil {
		http.Error(w, fmt.Sprintf("500 Internal Server Error"), http.StatusInternalServerError)
		logger.WithField("response.status", http.StatusInternalServerError).Error(err)

		return
	}

	w.WriteHeader(http.StatusOK)
	w.Write(wBytes)

	logger.WithField("response.status", http.StatusOK).Info("found compiled release")
}
<|endoftext|>


<commit_before>package gec

import (
	"fmt"
	"os"
	"testing"
)

func TestMain(m *testing.M) {
	os.Exit(run(m))
}

func run(m *testing.M) int {
	setup()
	defer teardown()
	code := m.Run()
	return code
}

var opt *Option
var tp *TextProcessor

func setup() {
	opt = NewOption()
	tp = NewTextProcessor(opt)
}

func teardown() {
}

func TestZenkakuASCII2HankakuASCIILowerAlphabet(t *testing.T) {
	actual := tp.zenkaku2Hankaku("ａｂｃｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚ")
	expected := "abcdefghijklmnopqrstuvwxyz"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestZenkakuASCII2HankakuASCIIUpperAlphabet(t *testing.T) {
	actual := tp.zenkaku2Hankaku("ＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺ")
	expected := "ABCDEFGHIJKLMNOPQRSTUVWXYZ"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestZenkakuASCII2HankakuASCIINumber(t *testing.T) {
	actual := tp.zenkaku2Hankaku("０１２３４５６７８９")
	expected := "0123456789"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestZenkakuASCII2HankakuASCIISymbol(t *testing.T) {
	actual := tp.zenkaku2Hankaku("：；＜＝＞？＠［＼］＾＿｀｛｜｝～！＂＃＄％＆＇（）＊，－．／a")
	expected := ":;<=>?@[\\]^_`{|}~!\"#$%&'()*,-./a"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestZenkakuSpace2HankakuSpace(t *testing.T) {
	actual := tp.zenkaku2Hankaku("　")
	expected := " "

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestCharref2Ascii(t *testing.T) {
	actual := tp.charref2Ascii(" <>&«»")
	expected := " <>&«»"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestRemoveKeisen(t *testing.T) {
	for r := 0x2500; r <= 0x257F; r++ {
		actual := tp.eliminateRuledLine(fmt.Sprintf("%c", r))
		expected := ""

		if actual != expected {
			t.Errorf("expected %v, but got %v", expected, actual)
		}
	}
}

func TestEliminateUselessTags(t *testing.T) {
	actuals := []string{
		tp.EliminateUselessTags("<select><option>a</option></select>foo"),
		tp.EliminateUselessTags(""),
		tp.EliminateUselessTags("<!abc 123>"),
		tp.EliminateUselessTags("
hoge
"),
		tp.EliminateUselessTags("
hoge
"),
		tp.EliminateUselessTags("
hoge
"),
	}
	expecteds := []string{
		"",
		"",
		"",
		"",
		"",
		"",
	}

	for i, _ := range actuals {
		if actuals[i] != expecteds[i] {
			t.Errorf("expected %v, but got %v", expecteds[i], actuals[i])
		}
	}
}

func TestHasOnlyTags(t *testing.T) {
	actuals := []bool{
		tp.IsOnlyTags("\t
 \n"),
		tp.IsOnlyTags("\t hoge \n"),
	}
	expecteds := []bool{
		true,
		false,
	}

	for i, _ := range actuals {
		if actuals[i] != expecteds[i] {
			t.Errorf("expected %v, but got %v", expecteds[i], actuals[i])
		}
	}
}

func TestEliminateLink(t *testing.T) {
	actuals := []string{
		tp.EliminateLink("
content

1.hoge

2.fuga

"),
		tp.EliminateLink("
content
hogefuga

1.hoge

2.fuga

"),
	}
	expecteds := []string{
		"content1.hoge2.fuga",
		"",
	}

	for i, _ := range actuals {
		if actuals[i] != expecteds[i] {
			t.Errorf("expected %v, but got %v", expecteds[i], actuals[i])
		}
	}
}

func TestIsLinklist(t *testing.T) {
	actuals := []bool{
		tp.isLinkList("

1.hoge

2.fuga

"),
		tp.isLinkList("start 123

1.hoge

2.fuga

456 end"),
	}
	expecteds := []bool{
		true,
		false,
	}

	for i, _ := range actuals {
		if actuals[i] != expecteds[i] {
			t.Errorf("expected %v, but got %v", expecteds[i], actuals[i])
		}
	}
}

func TestStripTags(t *testing.T) {
	actual := tp.EliminateTags("Ａ　ＢＣ\342\224\200
\n
<> \t\n ", "")
	expected := "A BC\n<> \n"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}
<commit_msg>fix text_processor test<commit_after>package gec

import (
	"fmt"
	"testing"
)

func TestZenkakuASCII2HankakuASCIILowerAlphabet(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actual := tp.zenkaku2Hankaku("ａｂｃｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚ")
	expected := "abcdefghijklmnopqrstuvwxyz"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestZenkakuASCII2HankakuASCIIUpperAlphabet(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actual := tp.zenkaku2Hankaku("ＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺ")
	expected := "ABCDEFGHIJKLMNOPQRSTUVWXYZ"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestZenkakuASCII2HankakuASCIINumber(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actual := tp.zenkaku2Hankaku("０１２３４５６７８９")
	expected := "0123456789"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestZenkakuASCII2HankakuASCIISymbol(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actual := tp.zenkaku2Hankaku("：；＜＝＞？＠［＼］＾＿｀｛｜｝～！＂＃＄％＆＇（）＊，－．／a")
	expected := ":;<=>?@[\\]^_`{|}~!\"#$%&'()*,-./a"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestZenkakuSpace2HankakuSpace(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actual := tp.zenkaku2Hankaku("　")
	expected := " "

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestCharref2Ascii(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actual := tp.charref2Ascii(" <>&«»")
	expected := " <>&«»"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

func TestRemoveKeisen(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	for r := 0x2500; r <= 0x257F; r++ {
		actual := tp.eliminateRuledLine(fmt.Sprintf("%c", r))
		expected := ""

		if actual != expected {
			t.Errorf("expected %v, but got %v", expected, actual)
		}
	}
}

func TestEliminateUselessTags(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actuals := []string{
		tp.EliminateUselessTags("<select><option>a</option></select>foo"),
		tp.EliminateUselessTags(""),
		tp.EliminateUselessTags("<!abc 123>"),
		tp.EliminateUselessTags("
hoge
"),
		tp.EliminateUselessTags("
hoge
"),
		tp.EliminateUselessTags("
hoge
"),
	}
	expecteds := []string{
		"",
		"",
		"",
		"",
		"",
		"",
	}

	for i, _ := range actuals {
		if actuals[i] != expecteds[i] {
			t.Errorf("expected %v, but got %v", expecteds[i], actuals[i])
		}
	}
}

func TestHasOnlyTags(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actuals := []bool{
		tp.IsOnlyTags("\t
 \n"),
		tp.IsOnlyTags("\t hoge \n"),
	}
	expecteds := []bool{
		true,
		false,
	}

	for i, _ := range actuals {
		if actuals[i] != expecteds[i] {
			t.Errorf("expected %v, but got %v", expecteds[i], actuals[i])
		}
	}
}

func TestEliminateLink(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actuals := []string{
		tp.EliminateLink("
content

1.hoge

2.fuga

"),
		tp.EliminateLink("
content
hogefuga

1.hoge

2.fuga

"),
	}
	expecteds := []string{
		"content1.hoge2.fuga",
		"",
	}

	for i, _ := range actuals {
		if actuals[i] != expecteds[i] {
			t.Errorf("expected %v, but got %v", expecteds[i], actuals[i])
		}
	}
}

func TestIsLinklist(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actuals := []bool{
		tp.isLinkList("

1.hoge

2.fuga

"),
		tp.isLinkList("start 123

1.hoge

2.fuga

456 end"),
	}
	expecteds := []bool{
		true,
		false,
	}

	for i, _ := range actuals {
		if actuals[i] != expecteds[i] {
			t.Errorf("expected %v, but got %v", expecteds[i], actuals[i])
		}
	}
}

func TestStripTags(t *testing.T) {
	opt := NewOption()
	tp := NewTextProcessor(opt)

	actual := tp.EliminateTags("Ａ　ＢＣ\342\224\200
\n
<> \t\n ", "")
	expected := "A BC\n<> \n"

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}

/*
func TestEvaluatelist(t *testing.T) {
	actual := tp.caluculateLinkRate([]string{"hoge", "hoge", "fuga", "foo"})
	expected := 6.0625

	if actual != expected {
		t.Errorf("expected %v, but got %v", expected, actual)
	}
}
*/
<|endoftext|>


<commit_before>/* 解析http请求的各种条件 */
package ogo

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

const (
	_DEF_PAGE     = 1 //1-base
	_DEF_PER_PAGE = 25
	_MAX_PER_PAGE = 100 //每页最大个数

	//time
	_DATE_FORM  = "2006-01-02"
	_DATE_FORM1 = "20060102"
)

//时间段
type TimeRange struct {
	Start time.Time
	End   time.Time
}

//order by
type OrderBy struct {
	Field string
	Sort  string
}

// 分页信息
type Pagination struct {
	Page    int
	PerPage int
	Offset  int
}

// 条件信息
type Condition struct {
	Field string
	Is    interface{}
	Not   interface{}
	Like  interface{}
	Join  interface{}
}

type Conditions map[string]*Condition

/* {{{ func NewPagation(page, perPage string) (p *Pagination)
 */
func NewPagination(page, perPage string) (p *Pagination) {
	var pageNum, offset, perNum int
	if page == "" {
		pageNum = _DEF_PAGE
	} else {
		pageNum, _ = strconv.Atoi(page)
		if pageNum < 1 {
			pageNum = _DEF_PAGE
		}
	}
	if perPage == "" {
		perNum = _DEF_PER_PAGE
	} else {
		perNum, _ = strconv.Atoi(perPage)
		if perNum > _MAX_PER_PAGE {
			perNum = _MAX_PER_PAGE
		}
	}
	offset = (pageNum - 1) * perNum
	p = &Pagination{
		Page:    pageNum,
		PerPage: perNum,
		Offset:  offset,
	}
	return
}

/* }}} */

/* {{{ func (rc *RESTContext) GetCondition(k string) (con *Condition, err error)
 * 设置参数查询条件
 */
func (rc *RESTContext) GetCondition(k string) (con *Condition, err error) {
	if cs, ok := rc.Env[ConditionsKey]; !ok {
		//没有conditions,自动初始化
		rc.SetEnv(ConditionsKey, make(Conditions))
		return nil, fmt.Errorf("Not found conditions! %s", ConditionsKey)
	} else {
		conditions := cs.(Conditions)
		if _, ok := conditions[k]; !ok {
			return nil, fmt.Errorf("Not found condition: %s", k)
		} else {
			con = conditions[k]
		}
	}
	return
}

/* }}} */

/* {{{ func (rc *RESTContext) setCondition(k string, con *Condition) (err error) {
	return
 *
*/
func (rc *RESTContext) setCondition(k string, con *Condition) {
	if k != "" {
		if _, ok := rc.Env[ConditionsKey]; !ok {
			//没有conditions,自动初始化
			rc.SetEnv(ConditionsKey, make(Conditions))
		}
		rc.Env[ConditionsKey].(Conditions)[k] = con
	}
}

/* }}} */

/* {{{ func (rc *RESTContext) setTimeRangeFromDate(p []string) {
 * 时间段信息
 */
func (rc *RESTContext) setTimeRangeFromDate(p []string) {
	tr := new(TimeRange)
	rc.SetEnv(TimeRangeKey, tr)

	var s, e, format string
	if len(p) > 1 { //有多个,第一个是start, 第二个是end, 其余忽略
		s, e = p[0], p[1]
	} else if len(p) > 0 { //只有一个, 可通过 "{start},{end}"方式传
		pieces := strings.SplitN(p[0], ",", 2)
		s = pieces[0]
		if len(pieces) > 1 {
			e = pieces[1]
		}
	}
	if len(s) == len(_DATE_FORM) {
		format = _DATE_FORM
	} else if len(s) == len(_DATE_FORM1) {
		format = _DATE_FORM1
	}
	if ts, err := time.ParseInLocation(format, s, Env().Location); err == nil {
		tr.Start = ts
		dura, _ := time.ParseDuration("86399s") // 一天少一秒
		tr.End = ts.Add(dura)                   //当天的最后一秒
		//只有成功获取了start, end才有意义
		if t, err := time.ParseInLocation(format, e, Env().Location); err == nil {
			te := t.Add(dura)
			if te.After(ts) { //必须比开始大
				tr.End = te
			}
		}
	}

	return
}

/* }}} */

/* {{{ func (rc *RESTContext) setTimeRangeFromStartEnd() {
 * 时间段信息
 */
func (rc *RESTContext) setTimeRangeFromStartEnd() {
	var sp, ep []string
	var ok bool
	r := rc.Request
	if sp, ok = r.Form[_PARAM_START]; !ok {
		//没有传入start,do nothing
		return
	}
	//删除
	r.Form.Del(_PARAM_START)

	if ep, ok = r.Form[_PARAM_END]; !ok {
		//没有传入end,do nothing
		return
	}
	//删除
	r.Form.Del(_PARAM_END)

	s, e := sp[0], ep[0]

	if len(s) != len(e) {
		//长度不一致,返回
		return
	}

	var format string
	if len(s) == len(_DATE_FORM) {
		format = _DATE_FORM
	} else if len(s) == len(_DATE_FORM1) {
		format = _DATE_FORM1
	}
	tr := new(TimeRange)
	if ts, err := time.ParseInLocation(format, s, Env().Location); err != nil {
		return
	} else {
		tr.Start = ts
	}
	if te, err := time.ParseInLocation(format, e, Env().Location); err != nil {
		return
	} else {
		tr.End = te
	}

	rc.SetEnv(TimeRangeKey, tr)

	return
}

/* }}} */

/* {{{ func (rc *RESTContext) setOrderBy(p string) {
 * 时间段信息
 */
func (rc *RESTContext) setOrderBy(p []string) {
	ob := new(OrderBy)
	rc.SetEnv(OrderByKey, ob)
	if len(p) > 0 { //只有一个, 可通过 "{start},{end}"方式传
		pieces := strings.SplitN(p[0], ",", 2)
		ob.Field = pieces[0]
		ob.Sort = "DESC" //默认降序
		if len(pieces) > 1 {
			if strings.ToUpper(pieces[1]) == "ASC" {
				ob.Sort = "ASC"
			}
		}
	}

	return
}

/* }}} */
<commit_msg>update<commit_after>/* 解析http请求的各种条件 */
package ogo

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

const (
	_DEF_PAGE     = 1 //1-base
	_DEF_PER_PAGE = 25
	_MAX_PER_PAGE = 100 //每页最大个数

	//time
	_DATE_FORM  = "2006-01-02"
	_DATE_FORM1 = "20060102"
)

//时间段
type TimeRange struct {
	Start time.Time
	End   time.Time
}

//order by
type OrderBy struct {
	Field string
	Sort  string
}

// 分页信息
type Pagination struct {
	Page    int
	PerPage int
	Offset  int
}

// 条件信息
type Condition struct {
	Field string
	Is    interface{}
	Not   interface{}
	Like  interface{}
	Join  interface{}
}

type Conditions map[string]*Condition

/* {{{ func NewPagation(page, perPage string) (p *Pagination)
 */
func NewPagination(page, perPage string) (p *Pagination) {
	var pageNum, offset, perNum int
	if page == "" {
		pageNum = _DEF_PAGE
	} else {
		pageNum, _ = strconv.Atoi(page)
		if pageNum < 1 {
			pageNum = _DEF_PAGE
		}
	}
	if perPage == "" {
		perNum = _DEF_PER_PAGE
	} else {
		perNum, _ = strconv.Atoi(perPage)
		if perNum > _MAX_PER_PAGE {
			perNum = _MAX_PER_PAGE
		}
	}
	offset = (pageNum - 1) * perNum
	p = &Pagination{
		Page:    pageNum,
		PerPage: perNum,
		Offset:  offset,
	}
	return
}

/* }}} */

/* {{{ func (rc *RESTContext) GetCondition(k string) (con *Condition, err error)
 * 设置参数查询条件
 */
func (rc *RESTContext) GetCondition(k string) (con *Condition, err error) {
	if cs, ok := rc.Env[ConditionsKey]; !ok {
		//没有conditions,自动初始化
		rc.SetEnv(ConditionsKey, make(Conditions))
		return nil, fmt.Errorf("Not found conditions! %s", ConditionsKey)
	} else {
		conditions := cs.(Conditions)
		if _, ok := conditions[k]; !ok {
			return nil, fmt.Errorf("Not found condition: %s", k)
		} else {
			con = conditions[k]
		}
	}
	return
}

/* }}} */

/* {{{ func (rc *RESTContext) setCondition(k string, con *Condition) (err error) {
	return
 *
*/
func (rc *RESTContext) setCondition(k string, con *Condition) {
	if k != "" {
		if _, ok := rc.Env[ConditionsKey]; !ok {
			//没有conditions,自动初始化
			rc.SetEnv(ConditionsKey, make(Conditions))
		}
		rc.Env[ConditionsKey].(Conditions)[k] = con
	}
}

/* }}} */

/* {{{ func (rc *RESTContext) setTimeRangeFromDate(p []string) {
 * 时间段信息
 */
func (rc *RESTContext) setTimeRangeFromDate(p []string) {
	tr := new(TimeRange)
	rc.SetEnv(TimeRangeKey, tr)

	var s, e, format string
	if len(p) > 1 { //有多个,第一个是start, 第二个是end, 其余忽略
		s, e = p[0], p[1]
	} else if len(p) > 0 { //只有一个, 可通过 "{start},{end}"方式传
		pieces := strings.SplitN(p[0], ",", 2)
		s = pieces[0]
		if len(pieces) > 1 {
			e = pieces[1]
		}
	}
	if len(s) == len(_DATE_FORM) {
		format = _DATE_FORM
	} else if len(s) == len(_DATE_FORM1) {
		format = _DATE_FORM1
	}
	if ts, err := time.ParseInLocation(format, s, Env().Location); err == nil {
		tr.Start = ts
		dura, _ := time.ParseDuration("86399s") // 一天少一秒
		tr.End = ts.Add(dura)                   //当天的最后一秒
		//只有成功获取了start, end才有意义
		if t, err := time.ParseInLocation(format, e, Env().Location); err == nil {
			te := t.Add(dura)
			if te.After(ts) { //必须比开始大
				tr.End = te
			}
		}
	}

	return
}

/* }}} */

/* {{{ func (rc *RESTContext) setTimeRangeFromStartEnd() {
 * 时间段信息
 */
func (rc *RESTContext) setTimeRangeFromStartEnd() {
	var sp, ep []string
	var ok bool
	r := rc.Request
	if sp, ok = r.Form[_PARAM_START]; !ok {
		//没有传入start,do nothing
		return
	}
	//删除
	r.Form.Del(_PARAM_START)

	if ep, ok = r.Form[_PARAM_END]; !ok {
		//没有传入end,do nothing
		return
	}
	//删除
	r.Form.Del(_PARAM_END)

	s, e := sp[0], ep[0]

	if len(s) != len(e) {
		//长度不一致,返回
		return
	}

	var format string
	if len(s) == len(_DATE_FORM) {
		format = _DATE_FORM
	} else if len(s) == len(_DATE_FORM1) {
		format = _DATE_FORM1
	}
	tr := new(TimeRange)
	if ts, err := time.ParseInLocation(format, s, Env().Location); err != nil {
		return
	} else {
		tr.Start = ts
	}
	if te, err := time.ParseInLocation(format, e, Env().Location); err != nil {
		return
	} else {
		dura, _ := time.ParseDuration("86399s") // 一天少一秒
		tr.End = te.Add(dura)                   //当天的最后一秒
	}

	rc.SetEnv(TimeRangeKey, tr)

	return
}

/* }}} */

/* {{{ func (rc *RESTContext) setOrderBy(p string) {
 * 时间段信息
 */
func (rc *RESTContext) setOrderBy(p []string) {
	ob := new(OrderBy)
	rc.SetEnv(OrderByKey, ob)
	if len(p) > 0 { //只有一个, 可通过 "{start},{end}"方式传
		pieces := strings.SplitN(p[0], ",", 2)
		ob.Field = pieces[0]
		ob.Sort = "DESC" //默认降序
		if len(pieces) > 1 {
			if strings.ToUpper(pieces[1]) == "ASC" {
				ob.Sort = "ASC"
			}
		}
	}

	return
}

/* }}} */
<|endoftext|>


<commit_before>package parse

import (
	"fmt"
	"strconv"

	"github.com/tisp-lang/tisp/src/lib/ast"
	"github.com/tisp-lang/tisp/src/lib/debug"
	"github.com/tisp-lang/tisp/src/lib/parse/comb"
)

const (
	commentChar     = ';'
	importString    = "import"
	invalidChars    = "\x00"
	letString       = "let"
	mutualRecString = "mr"
	matchString     = "match"
	spaceChars      = " \t\n\r"
	specialChars    = "()[]{}\"'`$"
)

var reserveds = map[string]bool{
	"macro":         true,
	importString:    true,
	letString:       true,
	matchString:     true,
	mutualRecString: true,
}

// MainModule parses a main module file into an AST.
func MainModule(file, source string) ([]interface{}, error) {
	m, err := newState(file, source).mainModule()()

	if err != nil {
		return nil, err
	}

	return m.([]interface{}), nil
}

// SubModule parses a sub module file into an AST.
func SubModule(file, source string) ([]interface{}, error) {
	m, err := newState(file, source).subModule()()

	if err != nil {
		return nil, err
	}

	return m.([]interface{}), nil
}

func (s *state) mainModule() comb.Parser {
	return s.module(s.importModule(), s.let(), s.output())
}

func (s *state) subModule() comb.Parser {
	return s.module(s.importModule(), s.let())
}

func (s *state) module(ps ...comb.Parser) comb.Parser {
	return s.Exhaust(s.Prefix(s.blank(), s.Many(s.Or(ps...))))
}

func (s *state) importModule() comb.Parser {
	return s.withInfo(
		s.list(s.strippedString(importString), s.stringLiteral()),
		func(x interface{}, i debug.Info) (interface{}, error) {
			xs := x.([]interface{})

			path, err := strconv.Unquote(xs[1].(string))
			if err != nil {
				panic(err)
			}

			return ast.NewImport(path, i), nil
		})
}

func (s *state) let() comb.Parser {
	return s.Lazy(s.strictLet)
}

func (s *state) strictLet() comb.Parser {
	return s.Or(s.letVar(), s.letFunction(), s.mutuallyRecursiveLetFunctions())
}

func (s *state) letVar() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		return ast.NewLetVar(xs[1].(string), xs[2])
	}, s.list(s.strippedString(letString), s.identifier(), s.expression()))
}

func (s *state) letFunction() comb.Parser {
	return s.withInfo(
		s.list(
			s.strippedString(letString),
			s.list(s.identifier(), s.signature()),
			s.Many(s.let()),
			s.expression()),
		func(x interface{}, i debug.Info) (interface{}, error) {
			xs := x.([]interface{})
			ys := xs[1].([]interface{})
			return ast.NewLetFunction(ys[0].(string), ys[1].(ast.Signature), xs[2].([]interface{}), xs[3], i), nil
		})
}

func (s *state) optionalArgument() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		return ast.NewOptionalArgument(xs[0].(string), xs[1])
	}, s.strip(s.list(s.identifier(), s.expression())))
}

func (s *state) expandedArgument() comb.Parser {
	return s.strip(s.expanded(s.identifier()))
}

func (s *state) halfSignature() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})

		xs0 := xs[0].([]interface{})
		requireds := make([]string, len(xs0))
		for i, r := range xs0 {
			requireds[i] = r.(string)
		}

		xs1 := xs[1].([]interface{})
		optionals := make([]ast.OptionalArgument, len(xs1))
		for i, o := range xs1 {
			optionals[i] = o.(ast.OptionalArgument)
		}

		expanded := ""
		if xs[2] != nil {
			expanded = xs[2].(string)
		}

		return [3]interface{}{requireds, optionals, expanded}
	}, s.And(s.Many(s.identifier()), s.Many(s.optionalArgument()), s.Maybe(s.expandedArgument())))
}

func (s *state) signature() comb.Parser {

	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})

		pas := xs[0].([3]interface{})
		kas, ok := xs[1].([3]interface{})

		if !ok {
			kas = [3]interface{}{([]string)(nil), ([]ast.OptionalArgument)(nil), ""}
		}

		return ast.NewSignature(
			pas[0].([]string), pas[1].([]ast.OptionalArgument), pas[2].(string),
			kas[0].([]string), kas[1].([]ast.OptionalArgument), kas[2].(string),
		)
	}, s.And(s.halfSignature(), s.Maybe(s.Prefix(s.strippedString("."), s.halfSignature()))))
}

func (s *state) output() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		expanded := false

		if xs[0] != nil {
			expanded = true
		}

		return ast.NewOutput(xs[1], expanded)
	}, s.And(s.Maybe(s.String("..")), s.expression()))
}

func (s *state) expanded(p comb.Parser) comb.Parser {
	return s.Prefix(s.String(".."), p)
}

func (s *state) expression() comb.Parser {
	return s.Lazy(s.strictExpression)
}

func (s *state) strictExpression() comb.Parser {
	return s.strip(s.Or(
		s.identifier(),
		s.stringLiteral(),
		s.match(),
		s.app(),
		s.listLiteral(),
		s.dictLiteral(),
		// s.appFunc("lambda", s.sequence("'(", ")")),
	))
}

func (s *state) listLiteral() comb.Parser {
	return s.appFunc("$list", s.sequence("[", "]"))
}

func (s *state) dictLiteral() comb.Parser {
	return s.appFunc("$dict", s.sequence("{", "}"))
}

func (s *state) match() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		ks := xs[2].([]interface{})

		cs := make([]ast.MatchCase, 0, len(ks))

		for _, k := range ks {
			xs := k.([]interface{})
			cs = append(cs, ast.NewMatchCase(xs[0], xs[1]))
		}

		return ast.NewMatch(xs[1], cs)
	}, s.list(
		s.strippedString(matchString),
		s.expression(),
		s.Many1(s.And(s.pattern(), s.expression()))))
}

func (s *state) pattern() comb.Parser {
	return s.Or(
		s.identifier(),
		s.stringLiteral(),
		s.listLiteral(),
		s.dictLiteral())
}

func (s *state) mutuallyRecursiveLetFunctions() comb.Parser {
	return s.withInfo(
		s.list(s.strippedString(mutualRecString), s.Many(s.letFunction())),
		func(x interface{}, i debug.Info) (interface{}, error) {
			xs := x.([]interface{})[1].([]interface{})
			fs := make([]ast.LetFunction, len(xs))

			for i, l := range xs {
				fs[i] = l.(ast.LetFunction)
			}

			return ast.NewMutualRecursion(fs, i), nil
		})
}

func (s *state) app() comb.Parser {
	return s.appWithInfo(
		s.list(s.expression(), s.arguments()),
		func(x interface{}) (interface{}, ast.Arguments) {
			xs := x.([]interface{})
			return xs[0], xs[1].(ast.Arguments)
		})
}

func (s *state) appWithInfo(p comb.Parser, f func(interface{}) (interface{}, ast.Arguments)) comb.Parser {
	return s.withInfo(
		p,
		func(x interface{}, i debug.Info) (interface{}, error) {
			f, args := f(x)
			return ast.NewApp(f, args, i), nil
		})
}

func (s *state) withInfo(p comb.Parser, f func(interface{}, debug.Info) (interface{}, error)) comb.Parser {
	return func() (interface{}, error) {
		i := s.debugInfo()
		x, err := p()

		if err != nil {
			return nil, err
		}

		return f(x, i)
	}
}

func (s *state) arguments() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})

		ks := ([]ast.KeywordArgument)(nil)
		dicts := []interface{}{}

		if xs, ok := xs[1].([2]interface{}); ok {
			ks = xs[0].([]ast.KeywordArgument)
			dicts = xs[1].([]interface{})
		}

		return ast.NewArguments(xs[0].([]ast.PositionalArgument), ks, dicts)
	}, s.And(
		s.positionalArguments(),
		s.Maybe(s.Prefix(s.strippedString("."), s.keywordArguments()))))
}

func (s *state) positionalArguments() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		ps := make([]ast.PositionalArgument, len(xs))

		for i, x := range xs {
			ps[i] = x.(ast.PositionalArgument)
		}

		return ps
	}, s.Many(s.positionalArgument()))
}

func (s *state) positionalArgument() comb.Parser {
	unexpanded := s.App(func(x interface{}) interface{} {
		return ast.NewPositionalArgument(x, false)
	}, s.expression())

	expanded := s.App(func(x interface{}) interface{} {
		return ast.NewPositionalArgument(x, true)
	}, s.expanded(s.expression()))

	return s.Or(unexpanded, expanded)
}

func (s *state) keywordArguments() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})

		ys := xs[0].([]interface{})
		ks := make([]ast.KeywordArgument, len(ys))
		for i, y := range ys {
			ks[i] = y.(ast.KeywordArgument)
		}

		return [2]interface{}{ks, xs[1].([]interface{})}
	}, s.And(s.Many(s.keywordArgument()), s.Many(s.expanded(s.expression()))))
}

func (s *state) keywordArgument() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		return ast.NewKeywordArgument(xs[0].(string), xs[1])
	}, s.And(s.identifier(), s.expression()))
}

func (s *state) identifier() comb.Parser {
	cs := string(commentChar) + invalidChars + spaceChars + specialChars
	p := s.strip(s.Stringify(s.And(s.NotInString(cs+"."), s.Stringify(s.Many(s.NotInString(cs))))))

	return func() (interface{}, error) {
		x, err := p()

		if err != nil {
			return nil, err
		}

		if _, ok := reserveds[x.(string)]; ok {
			return nil, fmt.Errorf("%#v is a reserved identifier", x)
		}

		return x, nil
	}
}

func (s *state) stringLiteral() comb.Parser {
	c := s.Char('"')

	return s.Stringify(s.And(
		c,
		s.Many(s.Or(s.NotInString("\"\\"), s.String("\\\""), s.String("\\\\"))),
		s.strip(c)))
}

func (s *state) list(ps ...comb.Parser) comb.Parser {
	return s.stringWrap("(", s.And(ps...), ")")
}

func (s *state) sequence(l, r string) comb.Parser {
	return s.App(func(x interface{}) interface{} {
		return ast.NewArguments(x.([]ast.PositionalArgument), nil, nil)
	}, s.stringWrap(l, s.positionalArguments(), r))
}

func (s *state) stringWrap(l string, p comb.Parser, r string) comb.Parser {
	return s.Wrap(s.strippedString(l), p, s.strippedString(r))
}

func (s *state) appFunc(ident string, p comb.Parser) comb.Parser {
	return s.appWithInfo(
		p,
		func(x interface{}) (interface{}, ast.Arguments) {
			return ident, x.(ast.Arguments)
		})
}

func (s *state) strip(p comb.Parser) comb.Parser {
	b := s.blank()
	return s.Wrap(s.None(), p, b)
}

func (s *state) blank() comb.Parser {
	return s.Void(s.Many(s.Or(s.InString(spaceChars), s.comment())))
}

func (s *state) comment() comb.Parser {
	return s.Void(s.And(
		s.Char(commentChar),
		s.Many(s.NotChar('\n')), s.Char('\n')))
}

func (s *state) strippedString(str string) comb.Parser {
	return s.strip(s.String(str))
}
<commit_msg>Remove unused constants<commit_after>package parse

import (
	"fmt"
	"strconv"

	"github.com/tisp-lang/tisp/src/lib/ast"
	"github.com/tisp-lang/tisp/src/lib/debug"
	"github.com/tisp-lang/tisp/src/lib/parse/comb"
)

const (
	commentChar     = ';'
	importString    = "import"
	invalidChars    = "\x00"
	letString       = "let"
	mutualRecString = "mr"
	matchString     = "match"
	spaceChars      = " \t\n\r"
	specialChars    = "()[]{}\"$"
)

var reserveds = map[string]bool{
	importString:    true,
	letString:       true,
	matchString:     true,
	mutualRecString: true,
}

// MainModule parses a main module file into an AST.
func MainModule(file, source string) ([]interface{}, error) {
	m, err := newState(file, source).mainModule()()

	if err != nil {
		return nil, err
	}

	return m.([]interface{}), nil
}

// SubModule parses a sub module file into an AST.
func SubModule(file, source string) ([]interface{}, error) {
	m, err := newState(file, source).subModule()()

	if err != nil {
		return nil, err
	}

	return m.([]interface{}), nil
}

func (s *state) mainModule() comb.Parser {
	return s.module(s.importModule(), s.let(), s.output())
}

func (s *state) subModule() comb.Parser {
	return s.module(s.importModule(), s.let())
}

func (s *state) module(ps ...comb.Parser) comb.Parser {
	return s.Exhaust(s.Prefix(s.blank(), s.Many(s.Or(ps...))))
}

func (s *state) importModule() comb.Parser {
	return s.withInfo(
		s.list(s.strippedString(importString), s.stringLiteral()),
		func(x interface{}, i debug.Info) (interface{}, error) {
			xs := x.([]interface{})

			path, err := strconv.Unquote(xs[1].(string))
			if err != nil {
				panic(err)
			}

			return ast.NewImport(path, i), nil
		})
}

func (s *state) let() comb.Parser {
	return s.Lazy(s.strictLet)
}

func (s *state) strictLet() comb.Parser {
	return s.Or(s.letVar(), s.letFunction(), s.mutuallyRecursiveLetFunctions())
}

func (s *state) letVar() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		return ast.NewLetVar(xs[1].(string), xs[2])
	}, s.list(s.strippedString(letString), s.identifier(), s.expression()))
}

func (s *state) letFunction() comb.Parser {
	return s.withInfo(
		s.list(
			s.strippedString(letString),
			s.list(s.identifier(), s.signature()),
			s.Many(s.let()),
			s.expression()),
		func(x interface{}, i debug.Info) (interface{}, error) {
			xs := x.([]interface{})
			ys := xs[1].([]interface{})
			return ast.NewLetFunction(ys[0].(string), ys[1].(ast.Signature), xs[2].([]interface{}), xs[3], i), nil
		})
}

func (s *state) optionalArgument() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		return ast.NewOptionalArgument(xs[0].(string), xs[1])
	}, s.strip(s.list(s.identifier(), s.expression())))
}

func (s *state) expandedArgument() comb.Parser {
	return s.strip(s.expanded(s.identifier()))
}

func (s *state) halfSignature() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})

		xs0 := xs[0].([]interface{})
		requireds := make([]string, len(xs0))
		for i, r := range xs0 {
			requireds[i] = r.(string)
		}

		xs1 := xs[1].([]interface{})
		optionals := make([]ast.OptionalArgument, len(xs1))
		for i, o := range xs1 {
			optionals[i] = o.(ast.OptionalArgument)
		}

		expanded := ""
		if xs[2] != nil {
			expanded = xs[2].(string)
		}

		return [3]interface{}{requireds, optionals, expanded}
	}, s.And(s.Many(s.identifier()), s.Many(s.optionalArgument()), s.Maybe(s.expandedArgument())))
}

func (s *state) signature() comb.Parser {

	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})

		pas := xs[0].([3]interface{})
		kas, ok := xs[1].([3]interface{})

		if !ok {
			kas = [3]interface{}{([]string)(nil), ([]ast.OptionalArgument)(nil), ""}
		}

		return ast.NewSignature(
			pas[0].([]string), pas[1].([]ast.OptionalArgument), pas[2].(string),
			kas[0].([]string), kas[1].([]ast.OptionalArgument), kas[2].(string),
		)
	}, s.And(s.halfSignature(), s.Maybe(s.Prefix(s.strippedString("."), s.halfSignature()))))
}

func (s *state) output() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		expanded := false

		if xs[0] != nil {
			expanded = true
		}

		return ast.NewOutput(xs[1], expanded)
	}, s.And(s.Maybe(s.String("..")), s.expression()))
}

func (s *state) expanded(p comb.Parser) comb.Parser {
	return s.Prefix(s.String(".."), p)
}

func (s *state) expression() comb.Parser {
	return s.Lazy(s.strictExpression)
}

func (s *state) strictExpression() comb.Parser {
	return s.strip(s.Or(
		s.identifier(),
		s.stringLiteral(),
		s.match(),
		s.app(),
		s.listLiteral(),
		s.dictLiteral(),
		// s.appFunc("lambda", s.sequence("'(", ")")),
	))
}

func (s *state) listLiteral() comb.Parser {
	return s.appFunc("$list", s.sequence("[", "]"))
}

func (s *state) dictLiteral() comb.Parser {
	return s.appFunc("$dict", s.sequence("{", "}"))
}

func (s *state) match() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		ks := xs[2].([]interface{})

		cs := make([]ast.MatchCase, 0, len(ks))

		for _, k := range ks {
			xs := k.([]interface{})
			cs = append(cs, ast.NewMatchCase(xs[0], xs[1]))
		}

		return ast.NewMatch(xs[1], cs)
	}, s.list(
		s.strippedString(matchString),
		s.expression(),
		s.Many1(s.And(s.pattern(), s.expression()))))
}

func (s *state) pattern() comb.Parser {
	return s.Or(
		s.identifier(),
		s.stringLiteral(),
		s.listLiteral(),
		s.dictLiteral())
}

func (s *state) mutuallyRecursiveLetFunctions() comb.Parser {
	return s.withInfo(
		s.list(s.strippedString(mutualRecString), s.Many(s.letFunction())),
		func(x interface{}, i debug.Info) (interface{}, error) {
			xs := x.([]interface{})[1].([]interface{})
			fs := make([]ast.LetFunction, len(xs))

			for i, l := range xs {
				fs[i] = l.(ast.LetFunction)
			}

			return ast.NewMutualRecursion(fs, i), nil
		})
}

func (s *state) app() comb.Parser {
	return s.appWithInfo(
		s.list(s.expression(), s.arguments()),
		func(x interface{}) (interface{}, ast.Arguments) {
			xs := x.([]interface{})
			return xs[0], xs[1].(ast.Arguments)
		})
}

func (s *state) appWithInfo(p comb.Parser, f func(interface{}) (interface{}, ast.Arguments)) comb.Parser {
	return s.withInfo(
		p,
		func(x interface{}, i debug.Info) (interface{}, error) {
			f, args := f(x)
			return ast.NewApp(f, args, i), nil
		})
}

func (s *state) withInfo(p comb.Parser, f func(interface{}, debug.Info) (interface{}, error)) comb.Parser {
	return func() (interface{}, error) {
		i := s.debugInfo()
		x, err := p()

		if err != nil {
			return nil, err
		}

		return f(x, i)
	}
}

func (s *state) arguments() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})

		ks := ([]ast.KeywordArgument)(nil)
		dicts := []interface{}{}

		if xs, ok := xs[1].([2]interface{}); ok {
			ks = xs[0].([]ast.KeywordArgument)
			dicts = xs[1].([]interface{})
		}

		return ast.NewArguments(xs[0].([]ast.PositionalArgument), ks, dicts)
	}, s.And(
		s.positionalArguments(),
		s.Maybe(s.Prefix(s.strippedString("."), s.keywordArguments()))))
}

func (s *state) positionalArguments() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		ps := make([]ast.PositionalArgument, len(xs))

		for i, x := range xs {
			ps[i] = x.(ast.PositionalArgument)
		}

		return ps
	}, s.Many(s.positionalArgument()))
}

func (s *state) positionalArgument() comb.Parser {
	unexpanded := s.App(func(x interface{}) interface{} {
		return ast.NewPositionalArgument(x, false)
	}, s.expression())

	expanded := s.App(func(x interface{}) interface{} {
		return ast.NewPositionalArgument(x, true)
	}, s.expanded(s.expression()))

	return s.Or(unexpanded, expanded)
}

func (s *state) keywordArguments() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})

		ys := xs[0].([]interface{})
		ks := make([]ast.KeywordArgument, len(ys))
		for i, y := range ys {
			ks[i] = y.(ast.KeywordArgument)
		}

		return [2]interface{}{ks, xs[1].([]interface{})}
	}, s.And(s.Many(s.keywordArgument()), s.Many(s.expanded(s.expression()))))
}

func (s *state) keywordArgument() comb.Parser {
	return s.App(func(x interface{}) interface{} {
		xs := x.([]interface{})
		return ast.NewKeywordArgument(xs[0].(string), xs[1])
	}, s.And(s.identifier(), s.expression()))
}

func (s *state) identifier() comb.Parser {
	cs := string(commentChar) + invalidChars + spaceChars + specialChars
	p := s.strip(s.Stringify(s.And(s.NotInString(cs+"."), s.Stringify(s.Many(s.NotInString(cs))))))

	return func() (interface{}, error) {
		x, err := p()

		if err != nil {
			return nil, err
		}

		if _, ok := reserveds[x.(string)]; ok {
			return nil, fmt.Errorf("%#v is a reserved identifier", x)
		}

		return x, nil
	}
}

func (s *state) stringLiteral() comb.Parser {
	c := s.Char('"')

	return s.Stringify(s.And(
		c,
		s.Many(s.Or(s.NotInString("\"\\"), s.String("\\\""), s.String("\\\\"))),
		s.strip(c)))
}

func (s *state) list(ps ...comb.Parser) comb.Parser {
	return s.stringWrap("(", s.And(ps...), ")")
}

func (s *state) sequence(l, r string) comb.Parser {
	return s.App(func(x interface{}) interface{} {
		return ast.NewArguments(x.([]ast.PositionalArgument), nil, nil)
	}, s.stringWrap(l, s.positionalArguments(), r))
}

func (s *state) stringWrap(l string, p comb.Parser, r string) comb.Parser {
	return s.Wrap(s.strippedString(l), p, s.strippedString(r))
}

func (s *state) appFunc(ident string, p comb.Parser) comb.Parser {
	return s.appWithInfo(
		p,
		func(x interface{}) (interface{}, ast.Arguments) {
			return ident, x.(ast.Arguments)
		})
}

func (s *state) strip(p comb.Parser) comb.Parser {
	b := s.blank()
	return s.Wrap(s.None(), p, b)
}

func (s *state) blank() comb.Parser {
	return s.Void(s.Many(s.Or(s.InString(spaceChars), s.comment())))
}

func (s *state) comment() comb.Parser {
	return s.Void(s.And(
		s.Char(commentChar),
		s.Many(s.NotChar('\n')), s.Char('\n')))
}

func (s *state) strippedString(str string) comb.Parser {
	return s.strip(s.String(str))
}
<|endoftext|>