<commit_before>package util

import (
	"github.com/ghts/ghts/lib"
	dpd "github.com/ghts/ghts/lib/daily_price_data"
	"github.com/ghts/ghts/lib/krx_time"
	xt "github.com/ghts/ghts/xing/base"
	xing "github.com/ghts/ghts/xing/go"

	"bytes"
	"database/sql"
	"time"
)

func F당일_일일_가격정보_수집(db *sql.DB) (에러 error) {
	defer func() {
		lib.S예외처리{M에러: &에러}.S실행()

		if 에러 == nil {
			lib.F문자열_출력("** 당일 가격정보 수집 완료 **")
		}
	}()

	if krx.F한국증시_동시호가_시간임() || krx.F한국증시_정규_거래_시간임() {
		lib.F문자열_출력("장 중에는 정확한 데이터를 수집할 수 없습니다.")
		return
	}

	당일 := lib.F일자2정수(xing.F당일())
	한달전 := lib.F금일().Add(-30 * lib.P1일)
	종목코드_모음 := make([]string, 0)

	for _, 종목코드 := range xing.F종목코드_모음_전체() {
		s := new(dpd.S종목별_일일_가격정보_모음)
		s.DB읽기with시작일(db, 종목코드, 한달전)

		if s.M저장소[len(s.M저장소)-1].M일자 == 당일 {
			continue
		}

		// 당일 데이터 없는 경우에만 수집 대상에 추가.
		종목코드_모음 = append(종목코드_모음, 종목코드)
	}

	if len(종목코드_모음) == 0 {
		return
	}

	dpd.F일일_가격정보_테이블_생성(db)

	당일_가격정보_맵 := lib.F확인2(xing.TrT8407_현물_멀티_현재가_조회(종목코드_모음))

	for 종목코드, 값 := range 당일_가격정보_맵 {
		s := new(dpd.S일일_가격정보)
		s.M종목코드 = 종목코드
		s.M일자 = 당일
		s.M시가 = float64(값.M시가)
		s.M고가 = float64(값.M고가)
		s.M저가 = float64(값.M저가)
		s.M종가 = float64(값.M현재가)
		s.M거래량 = float64(값.M누적_거래량)

		종목별_일일_가격정보_모음 := lib.F확인2(dpd.New종목별_일일_가격정보_모음([]*dpd.S일일_가격정보{s}))
		lib.F확인1(종목별_일일_가격정보_모음.DB저장(db))
	}

	return nil
}

func F일개월_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	return F고정_기간_일일_가격정보_수집(db, 종목코드_모음, 31*lib.P1일)
}

func F일년_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	return F고정_기간_일일_가격정보_수집(db, 종목코드_모음, lib.P1년)
}

func F고정_기간_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string, 기간 time.Duration, 추가_인수 ...bool) (에러 error) {
	if len(종목코드_모음) == 0 {
		return nil
	}

	dpd.F일일_가격정보_테이블_생성(db)

	시작일 := lib.F금일().Add(-1 * 기간)

	출력_여부 := true
	if len(추가_인수) > 0 {
		출력_여부 = 추가_인수[0]
	}

	종목코드_맵 := make(map[string]lib.S비어있음) // 종목 순서를 랜덤화

	for _, 종목코드 := range 종목코드_모음 {
		종목코드_맵[종목코드] = lib.F비어있는_값()
	}

	i := 0

	for 종목코드 := range 종목코드_맵 {
		if lib.F공통_종료_채널_닫힘() {
			return nil
		}

		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, i, len(종목코드_맵), 출력_여부)
		i++

		lib.F대기(lib.P4초) // TR 한도 초과 관련.
	}

	return nil
}

func F일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string, 추가_인수 ...bool) (에러 error) {
	var 시작일, 마지막_저장일 time.Time
	var 종목별_일일_가격정보_모음 *dpd.S종목별_일일_가격정보_모음

	dpd.F일일_가격정보_테이블_생성(db)

	출력_여부 := lib.F조건부_참거짓(len(추가_인수) > 0, 추가_인수[0], true)
	출력_문자열_버퍼 := new(bytes.Buffer)

	for i, 종목코드 := range 종목코드_모음 {
		종목별_일일_가격정보_모음 = lib.F확인2(dpd.New종목별_일일_가격정보_모음_DB읽기(db, 종목코드))

		// 시작일 설정
		시작일 = lib.F지금().AddDate(-30, 0, 0)
		if 에러 == nil && len(종목별_일일_가격정보_모음.M저장소) > 0 {
			// lib.S종목별_일일_가격정보_모음 는 일자 순서로 정렬되어 있음.
			마지막_저장일 = 종목별_일일_가격정보_모음.M저장소[len(종목별_일일_가격정보_모음.M저장소)-1].G일자2()
			시작일 = 마지막_저장일.AddDate(0, 0, 1)
		}

		if 시작일.After(xing.F당일()) {
			//lib.F문자열_출력("%v [%v] : 최신 데이터 업데이트.", i, 종목코드)
			continue
		} else if 시작일.After(lib.F금일().AddDate(0, 0, -14)) {
			// 데이터 수량이 1개이나 100개이나 소요 시간은 비슷함.
			시작일 = lib.F금일().AddDate(0, 0, -14)
		}

		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, i, len(종목코드_모음), 출력_여부, 출력_문자열_버퍼)
	}

	lib.F문자열_출력(출력_문자열_버퍼.String())

	return nil
}

func f일일_가격정보_수집_도우미(db *sql.DB, 종목코드 string, 시작일 time.Time, i, 전체_수량 int, 출력_여부 bool, 버퍼 ...*bytes.Buffer) {
	var 종료일 time.Time

	// 종료일 설정
	if lib.F지금().After(xing.F당일().Add(15*lib.P1시간 + lib.P30분)) {
		종료일 = xing.F당일()
	} else {
		종료일 = xing.F전일()
	}

	// 시작일 오류 확인
	if 시작일 = lib.F2일자(시작일); 시작일.After(종료일) {
		return
	} else if 시작일.Equal(종료일) { // 시작일과 종료일이 같으면 수천 개의 데이터를 불러오는 현상이 있음.
		시작일 = 시작일.AddDate(0, 0, -1)
	}

	// 데이터 수집
	값_모음, 에러 := xing.TrT8413_현물_차트_일주월(종목코드, 시작일, 종료일, xt.P일주월_일)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	} else if len(값_모음) == 0 {
		lib.F체크포인트(i, 종목코드, "추가 저장할 데이터가 없음.")
		return // 추가 저장할 데이터가 없음.
	}

	일일_가격정보_슬라이스 := make([]*dpd.S일일_가격정보, len(값_모음))

	for i, 일일_데이터 := range 값_모음 {
		일일_가격정보_슬라이스[i] = dpd.New일일_가격정보(
			일일_데이터.M종목코드,
			일일_데이터.M일자,
			일일_데이터.M시가,
			일일_데이터.M고가,
			일일_데이터.M저가,
			일일_데이터.M종가,
			일일_데이터.M거래량)
	}

	if 출력_여부 && len(버퍼) > 0 && 버퍼[0] != nil {
		// 버퍼가 존재하면 버퍼에 출력
		버퍼[0].WriteString(lib.F2문자열("%v 일일 가격 정보 수집 (%v/%v) : %v %v~%v %v개\n",
			lib.F지금().Format("15:04"), i+1, 전체_수량,
			xing.F종목_식별_문자열(종목코드), 시작일.Format(lib.P일자_형식), 종료일.Format(lib.P일자_형식), len(값_모음)))
	} else if 출력_여부 {
		lib.F문자열_출력("%v 일일 가격 정보 수집 (%v/%v) %.1f%% : %v %v~%v %v개\n",
			lib.F지금().Format("15:04"), i+1, 전체_수량, float64(i+1)/float64(전체_수량)*100,
			xing.F종목_식별_문자열(종목코드), 시작일.Format(lib.P일자_형식), 종료일.Format(lib.P일자_형식), len(값_모음))
	} else if i > 0 && i%100 == 0 {
		lib.F문자열_출력("%v 일일 가격 정보 수집 (%v/%v) %.1f%%",
			lib.F지금().Format("15:04"), i+1, 전체_수량, float64(i+1)/float64(전체_수량)*100)
	}

	종목별_일일_가격정보_모음, 에러 := dpd.New종목별_일일_가격정보_모음(일일_가격정보_슬라이스)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	}

	lib.F확인1(종목별_일일_가격정보_모음.DB저장(db))
}
<commit_msg>F당일_일일_가격정보_수집() 버그 수정.<commit_after>package util

import (
	"github.com/ghts/ghts/lib"
	dpd "github.com/ghts/ghts/lib/daily_price_data"
	"github.com/ghts/ghts/lib/krx_time"
	xt "github.com/ghts/ghts/xing/base"
	xing "github.com/ghts/ghts/xing/go"

	"bytes"
	"database/sql"
	"time"
)

func F당일_일일_가격정보_수집(db *sql.DB) (에러 error) {
	defer func() {
		lib.S예외처리{M에러: &에러}.S실행()

		if 에러 == nil {
			lib.F문자열_출력("** 당일 가격정보 수집 완료 **")
		}
	}()

	if krx.F한국증시_동시호가_시간임() || krx.F한국증시_정규_거래_시간임() {
		lib.F문자열_출력("장 중에는 정확한 데이터를 수집할 수 없습니다.")
		return
	}

	당일 := lib.F일자2정수(xing.F당일())
	한달전 := lib.F금일().Add(-30 * lib.P1일)
	종목코드_모음 := make([]string, 0)

	for _, 종목코드 := range xing.F종목코드_모음_전체() {
		s := new(dpd.S종목별_일일_가격정보_모음)
		s.DB읽기with시작일(db, 종목코드, 한달전)

		if len(s.M저장소) > 0 && s.M저장소[len(s.M저장소)-1].M일자 == 당일 {
			continue
		}

		// 당일 데이터 없는 경우에만 수집 대상에 추가.
		종목코드_모음 = append(종목코드_모음, 종목코드)
	}

	if len(종목코드_모음) == 0 {
		return
	}

	dpd.F일일_가격정보_테이블_생성(db)

	당일_가격정보_맵 := lib.F확인2(xing.TrT8407_현물_멀티_현재가_조회(종목코드_모음))

	for 종목코드, 값 := range 당일_가격정보_맵 {
		s := new(dpd.S일일_가격정보)
		s.M종목코드 = 종목코드
		s.M일자 = 당일
		s.M시가 = float64(값.M시가)
		s.M고가 = float64(값.M고가)
		s.M저가 = float64(값.M저가)
		s.M종가 = float64(값.M현재가)
		s.M거래량 = float64(값.M누적_거래량)

		종목별_일일_가격정보_모음 := lib.F확인2(dpd.New종목별_일일_가격정보_모음([]*dpd.S일일_가격정보{s}))
		lib.F확인1(종목별_일일_가격정보_모음.DB저장(db))
	}

	return nil
}

func F일개월_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	return F고정_기간_일일_가격정보_수집(db, 종목코드_모음, 31*lib.P1일)
}

func F일년_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	return F고정_기간_일일_가격정보_수집(db, 종목코드_모음, lib.P1년)
}

func F고정_기간_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string, 기간 time.Duration, 추가_인수 ...bool) (에러 error) {
	if len(종목코드_모음) == 0 {
		return nil
	}

	dpd.F일일_가격정보_테이블_생성(db)

	시작일 := lib.F금일().Add(-1 * 기간)

	출력_여부 := true
	if len(추가_인수) > 0 {
		출력_여부 = 추가_인수[0]
	}

	종목코드_맵 := make(map[string]lib.S비어있음) // 종목 순서를 랜덤화

	for _, 종목코드 := range 종목코드_모음 {
		종목코드_맵[종목코드] = lib.F비어있는_값()
	}

	i := 0

	for 종목코드 := range 종목코드_맵 {
		if lib.F공통_종료_채널_닫힘() {
			return nil
		}

		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, i, len(종목코드_맵), 출력_여부)
		i++

		lib.F대기(lib.P4초) // TR 한도 초과 관련.
	}

	return nil
}

func F일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string, 추가_인수 ...bool) (에러 error) {
	var 시작일, 마지막_저장일 time.Time
	var 종목별_일일_가격정보_모음 *dpd.S종목별_일일_가격정보_모음

	dpd.F일일_가격정보_테이블_생성(db)

	출력_여부 := lib.F조건부_참거짓(len(추가_인수) > 0, 추가_인수[0], true)
	출력_문자열_버퍼 := new(bytes.Buffer)

	for i, 종목코드 := range 종목코드_모음 {
		종목별_일일_가격정보_모음 = lib.F확인2(dpd.New종목별_일일_가격정보_모음_DB읽기(db, 종목코드))

		// 시작일 설정
		시작일 = lib.F지금().AddDate(-30, 0, 0)
		if 에러 == nil && len(종목별_일일_가격정보_모음.M저장소) > 0 {
			// lib.S종목별_일일_가격정보_모음 는 일자 순서로 정렬되어 있음.
			마지막_저장일 = 종목별_일일_가격정보_모음.M저장소[len(종목별_일일_가격정보_모음.M저장소)-1].G일자2()
			시작일 = 마지막_저장일.AddDate(0, 0, 1)
		}

		if 시작일.After(xing.F당일()) {
			//lib.F문자열_출력("%v [%v] : 최신 데이터 업데이트.", i, 종목코드)
			continue
		} else if 시작일.After(lib.F금일().AddDate(0, 0, -14)) {
			// 데이터 수량이 1개이나 100개이나 소요 시간은 비슷함.
			시작일 = lib.F금일().AddDate(0, 0, -14)
		}

		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, i, len(종목코드_모음), 출력_여부, 출력_문자열_버퍼)
	}

	lib.F문자열_출력(출력_문자열_버퍼.String())

	return nil
}

func f일일_가격정보_수집_도우미(db *sql.DB, 종목코드 string, 시작일 time.Time, i, 전체_수량 int, 출력_여부 bool, 버퍼 ...*bytes.Buffer) {
	var 종료일 time.Time

	// 종료일 설정
	if lib.F지금().After(xing.F당일().Add(15*lib.P1시간 + lib.P30분)) {
		종료일 = xing.F당일()
	} else {
		종료일 = xing.F전일()
	}

	// 시작일 오류 확인
	if 시작일 = lib.F2일자(시작일); 시작일.After(종료일) {
		return
	} else if 시작일.Equal(종료일) { // 시작일과 종료일이 같으면 수천 개의 데이터를 불러오는 현상이 있음.
		시작일 = 시작일.AddDate(0, 0, -1)
	}

	// 데이터 수집
	값_모음, 에러 := xing.TrT8413_현물_차트_일주월(종목코드, 시작일, 종료일, xt.P일주월_일)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	} else if len(값_모음) == 0 {
		lib.F체크포인트(i, 종목코드, "추가 저장할 데이터가 없음.")
		return // 추가 저장할 데이터가 없음.
	}

	일일_가격정보_슬라이스 := make([]*dpd.S일일_가격정보, len(값_모음))

	for i, 일일_데이터 := range 값_모음 {
		일일_가격정보_슬라이스[i] = dpd.New일일_가격정보(
			일일_데이터.M종목코드,
			일일_데이터.M일자,
			일일_데이터.M시가,
			일일_데이터.M고가,
			일일_데이터.M저가,
			일일_데이터.M종가,
			일일_데이터.M거래량)
	}

	if 출력_여부 && len(버퍼) > 0 && 버퍼[0] != nil {
		// 버퍼가 존재하면 버퍼에 출력
		버퍼[0].WriteString(lib.F2문자열("%v 일일 가격 정보 수집 (%v/%v) : %v %v~%v %v개\n",
			lib.F지금().Format("15:04"), i+1, 전체_수량,
			xing.F종목_식별_문자열(종목코드), 시작일.Format(lib.P일자_형식), 종료일.Format(lib.P일자_형식), len(값_모음)))
	} else if 출력_여부 {
		lib.F문자열_출력("%v 일일 가격 정보 수집 (%v/%v) %.1f%% : %v %v~%v %v개\n",
			lib.F지금().Format("15:04"), i+1, 전체_수량, float64(i+1)/float64(전체_수량)*100,
			xing.F종목_식별_문자열(종목코드), 시작일.Format(lib.P일자_형식), 종료일.Format(lib.P일자_형식), len(값_모음))
	} else if i > 0 && i%100 == 0 {
		lib.F문자열_출력("%v 일일 가격 정보 수집 (%v/%v) %.1f%%",
			lib.F지금().Format("15:04"), i+1, 전체_수량, float64(i+1)/float64(전체_수량)*100)
	}

	종목별_일일_가격정보_모음, 에러 := dpd.New종목별_일일_가격정보_모음(일일_가격정보_슬라이스)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	}

	lib.F확인1(종목별_일일_가격정보_모음.DB저장(db))
}
<|endoftext|>


<commit_before>/*
 * This file is part of the libvirt-go-xml project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * Copyright (C) 2016 Red Hat, Inc.
 *
 */

package libvirtxml

import (
	"encoding/xml"
)

type DomainCaps struct {
	XMLName   xml.Name             `xml:"domainCapabilities"`
	Path      string               `xml:"path"`
	Domain    string               `xml:"domain"`
	Machine   string               `xml:"machine,omitempty"`
	Arch      string               `xml:"arch"`
	VCPU      *DomainCapsVCPU      `xml:"vcpu"`
	IOThreads *DomainCapsIOThreads `xml:"iothreads"`
	OS        *DomainCapsOS        `xml:"os"`
	CPU       *DomainCapsCPU       `xml:"cpu"`
	Devices   *DomainCapsDevices   `xml:"devices"`
	Features  *DomainCapsFeatures  `xml:"features"`
}

type DomainCapsVCPU struct {
	Max uint `xml:"max,attr"`
}

type DomainCapsOS struct {
	Supported string              `xml:"supported,attr"`
	Loader    *DomainCapsOSLoader `xml:"loader"`
	Enums     []DomainCapsEnum    `xml:"enum"`
}

type DomainCapsOSLoader struct {
	Supported string           `xml:"supported,attr"`
	Values    []string         `xml:"value"`
	Enums     []DomainCapsEnum `xml:"enum"`
}

type DomainCapsIOThreads struct {
	Supported string `xml:"supported,attr"`
}

type DomainCapsCPU struct {
	Modes []DomainCapsCPUMode `xml:"mode"`
}

type DomainCapsCPUMode struct {
	Name      string                 `xml:"name,attr"`
	Supported string                 `xml:"supported,attr"`
	Models    []DomainCapsCPUModel   `xml:"model"`
	Vendor    string                 `xml:"vendor,omitempty"`
	Features  []DomainCapsCPUFeature `xml:"feature"`
	Enums     []DomainCapsEnum       `xml:"enum"`
}

type DomainCapsCPUModel struct {
	Name     string `xml:",chardata"`
	Usable   string `xml:"usable,attr,omitempty"`
	Fallback string `xml:"fallback,attr,omitempty"`
}

type DomainCapsCPUFeature struct {
	Policy string `xml:"policy,attr,omitempty"`
	Name   string `xml:"name,attr"`
}

type DomainCapsEnum struct {
	Name   string   `xml:"name,attr"`
	Values []string `xml:"value"`
}

type DomainCapsDevices struct {
	Disk     *DomainCapsDevice `xml:"disk"`
	Graphics *DomainCapsDevice `xml:"graphics"`
	Video    *DomainCapsDevice `xml:"video"`
	HostDev  *DomainCapsDevice `xml:"hostdev"`
	RNG      *DomainCapsDevice `xml:"rng"`
}

type DomainCapsDevice struct {
	Supported string           `xml:"supported,attr"`
	Enums     []DomainCapsEnum `xml:"enum"`
}

type DomainCapsFeatures struct {
	GIC               *DomainCapsFeatureGIC               `xml:"gic"`
	VMCoreInfo        *DomainCapsFeatureVMCoreInfo        `xml:"vmcoreinfo"`
	GenID             *DomainCapsFeatureGenID             `xml:"genid"`
	BackingStoreInput *DomainCapsFeatureBackingStoreInput `xml:"backingStoreInput"`
	Backup            *DomainCapsFeatureBackup            `xml:"backup"`
	SEV               *DomainCapsFeatureSEV               `xml:"sev"`
}

type DomainCapsFeatureGIC struct {
	Supported string           `xml:"supported,attr"`
	Enums     []DomainCapsEnum `xml:"enum"`
}

type DomainCapsFeatureVMCoreInfo struct {
	Supported string `xml:"supported,attr"`
}

type DomainCapsFeatureGenID struct {
	Supported string `xml:"supported,attr"`
}

type DomainCapsFeatureBackingStoreInput struct {
	Supported string `xml:"supported,attr"`
}

type DomainCapsFeatureBackup struct {
	Supported string `xml:"supported,attr"`
}

type DomainCapsFeatureSEV struct {
	Supported       string `xml:"supported,attr"`
	CBitPos         uint   `xml:"cbitpos,omitempty"`
	ReducedPhysBits uint   `xml:"reducedPhysBits,omitempty"`
}

func (c *DomainCaps) Unmarshal(doc string) error {
	return xml.Unmarshal([]byte(doc), c)
}

func (c *DomainCaps) Marshal() (string, error) {
	doc, err := xml.MarshalIndent(c, "", "  ")
	if err != nil {
		return "", err
	}
	return string(doc), nil
}
<commit_msg>Parse CPU model deprecated attribute for domain capabilities<commit_after>/*
 * This file is part of the libvirt-go-xml project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * Copyright (C) 2016 Red Hat, Inc.
 *
 */

package libvirtxml

import (
	"encoding/xml"
)

type DomainCaps struct {
	XMLName   xml.Name             `xml:"domainCapabilities"`
	Path      string               `xml:"path"`
	Domain    string               `xml:"domain"`
	Machine   string               `xml:"machine,omitempty"`
	Arch      string               `xml:"arch"`
	VCPU      *DomainCapsVCPU      `xml:"vcpu"`
	IOThreads *DomainCapsIOThreads `xml:"iothreads"`
	OS        *DomainCapsOS        `xml:"os"`
	CPU       *DomainCapsCPU       `xml:"cpu"`
	Devices   *DomainCapsDevices   `xml:"devices"`
	Features  *DomainCapsFeatures  `xml:"features"`
}

type DomainCapsVCPU struct {
	Max uint `xml:"max,attr"`
}

type DomainCapsOS struct {
	Supported string              `xml:"supported,attr"`
	Loader    *DomainCapsOSLoader `xml:"loader"`
	Enums     []DomainCapsEnum    `xml:"enum"`
}

type DomainCapsOSLoader struct {
	Supported string           `xml:"supported,attr"`
	Values    []string         `xml:"value"`
	Enums     []DomainCapsEnum `xml:"enum"`
}

type DomainCapsIOThreads struct {
	Supported string `xml:"supported,attr"`
}

type DomainCapsCPU struct {
	Modes []DomainCapsCPUMode `xml:"mode"`
}

type DomainCapsCPUMode struct {
	Name      string                 `xml:"name,attr"`
	Supported string                 `xml:"supported,attr"`
	Models    []DomainCapsCPUModel   `xml:"model"`
	Vendor    string                 `xml:"vendor,omitempty"`
	Features  []DomainCapsCPUFeature `xml:"feature"`
	Enums     []DomainCapsEnum       `xml:"enum"`
}

type DomainCapsCPUModel struct {
	Name       string `xml:",chardata"`
	Usable     string `xml:"usable,attr,omitempty"`
	Fallback   string `xml:"fallback,attr,omitempty"`
	Deprecated string `xml:"deprecated,attr,omitempty"`
}

type DomainCapsCPUFeature struct {
	Policy string `xml:"policy,attr,omitempty"`
	Name   string `xml:"name,attr"`
}

type DomainCapsEnum struct {
	Name   string   `xml:"name,attr"`
	Values []string `xml:"value"`
}

type DomainCapsDevices struct {
	Disk     *DomainCapsDevice `xml:"disk"`
	Graphics *DomainCapsDevice `xml:"graphics"`
	Video    *DomainCapsDevice `xml:"video"`
	HostDev  *DomainCapsDevice `xml:"hostdev"`
	RNG      *DomainCapsDevice `xml:"rng"`
}

type DomainCapsDevice struct {
	Supported string           `xml:"supported,attr"`
	Enums     []DomainCapsEnum `xml:"enum"`
}

type DomainCapsFeatures struct {
	GIC               *DomainCapsFeatureGIC               `xml:"gic"`
	VMCoreInfo        *DomainCapsFeatureVMCoreInfo        `xml:"vmcoreinfo"`
	GenID             *DomainCapsFeatureGenID             `xml:"genid"`
	BackingStoreInput *DomainCapsFeatureBackingStoreInput `xml:"backingStoreInput"`
	Backup            *DomainCapsFeatureBackup            `xml:"backup"`
	SEV               *DomainCapsFeatureSEV               `xml:"sev"`
}

type DomainCapsFeatureGIC struct {
	Supported string           `xml:"supported,attr"`
	Enums     []DomainCapsEnum `xml:"enum"`
}

type DomainCapsFeatureVMCoreInfo struct {
	Supported string `xml:"supported,attr"`
}

type DomainCapsFeatureGenID struct {
	Supported string `xml:"supported,attr"`
}

type DomainCapsFeatureBackingStoreInput struct {
	Supported string `xml:"supported,attr"`
}

type DomainCapsFeatureBackup struct {
	Supported string `xml:"supported,attr"`
}

type DomainCapsFeatureSEV struct {
	Supported       string `xml:"supported,attr"`
	CBitPos         uint   `xml:"cbitpos,omitempty"`
	ReducedPhysBits uint   `xml:"reducedPhysBits,omitempty"`
}

func (c *DomainCaps) Unmarshal(doc string) error {
	return xml.Unmarshal([]byte(doc), c)
}

func (c *DomainCaps) Marshal() (string, error) {
	doc, err := xml.MarshalIndent(c, "", "  ")
	if err != nil {
		return "", err
	}
	return string(doc), nil
}
<|endoftext|>


<commit_before>package upload

import (
	"bytes"
	"net/http"
	"path/filepath"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/chadweimer/gomp/modules/conf"
)

// S3Driver is an implementation of Driver that uses the Amazon S3.
type S3Driver struct {
	cfg *conf.Config
}

func NewS3Driver(cfg *conf.Config) S3Driver {
	return S3Driver{cfg: cfg}
}

func (u S3Driver) connectToS3() *s3.S3 {
	awsConfig := &aws.Config{
		Region: aws.String(u.cfg.AwsRegion),
	}
	if u.cfg.AwsAccessKeyID != "" && u.cfg.AwsSecretAccessKey != "" {
		awsConfig.Credentials = credentials.NewStaticCredentials(u.cfg.AwsAccessKeyID, u.cfg.AwsSecretAccessKey, "")
	}
	return s3.New(session.New(awsConfig))
}

func (u S3Driver) Save(filePath string, data []byte) error {
	svc := u.connectToS3()

	bucket := u.cfg.UploadPath
	key := filepath.ToSlash(filePath)
	contentType := http.DetectContentType(data)
	_, err := svc.PutObject(&s3.PutObjectInput{
		Body:        bytes.NewReader(data),
		ContentType: &contentType,
		Bucket:      &bucket,
		Key:         &key,
	})
	return err
}

func (u S3Driver) Delete(filePath string) error {
	svc := u.connectToS3()

	bucket := u.cfg.UploadPath
	key := filepath.ToSlash(filePath)

	_, err := svc.DeleteObject(&s3.DeleteObjectInput{
		Bucket: &bucket,
		Key:    &key,
	})
	return err
}
func (u S3Driver) DeleteAll(dirPath string) error {

	svc := u.connectToS3()

	bucket := u.cfg.UploadPath
	prefix := filepath.ToSlash(dirPath)

	listOutput, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
		Bucket: &bucket,
		Prefix: &prefix,
	})
	if err != nil {
		return err
	}
	for _, object := range listOutput.Contents {
		_, err := svc.DeleteObject(&s3.DeleteObjectInput{
			Bucket: &bucket,
			Key:    object.Key,
		})
		if err != nil {
			return err
		}
	}

	return nil
}

func (u S3Driver) List(dirPath string) ([]string, []string, []string, error) {
	var names []string
	var origURLs []string
	var thumbURLs []string

	svc := u.connectToS3()

	bucket := u.cfg.UploadPath
	prefix := filepath.ToSlash(filepath.Join(dirPath, "images"))

	listOutput, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
		Bucket: &bucket,
		Prefix: &prefix,
	})
	if err != nil {
		return names, origURLs, thumbURLs, err
	}

	for _, object := range listOutput.Contents {
		origReq, _ := svc.GetObjectRequest(&s3.GetObjectInput{
			Bucket: &bucket,
			Key:    object.Key,
		})
		origURL, err := origReq.Presign(30 * time.Minute)
		if err == nil {
			urlSegments := strings.Split(*object.Key, "/")
			name := urlSegments[len(urlSegments)-1]

			names = append(names, name)
			origURLs = append(origURLs, origURL)

			thumbKey := strings.Replace(*object.Key, "/images/", "/thumbs/", 1)
			thumbReq, _ := svc.GetObjectRequest(&s3.GetObjectInput{
				Bucket: &bucket,
				Key:    &thumbKey,
			})
			thumbURL, err := thumbReq.Presign(30 * time.Minute)
			if err == nil {
				thumbURLs = append(thumbURLs, thumbURL)
			} else {
				thumbURLs = append(thumbURLs, "")
			}
		}
	}

	return names, origURLs, thumbURLs, nil
}
<commit_msg>Fixed a bug in setting the AWS region<commit_after>package upload

import (
	"bytes"
	"net/http"
	"path/filepath"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/chadweimer/gomp/modules/conf"
)

// S3Driver is an implementation of Driver that uses the Amazon S3.
type S3Driver struct {
	cfg *conf.Config
}

func NewS3Driver(cfg *conf.Config) S3Driver {
	return S3Driver{cfg: cfg}
}

func (u S3Driver) connectToS3() *s3.S3 {
	awsConfig := new(aws.Config)
	if u.cfg.AwsRegion != "" {
		awsConfig.Region = &u.cfg.AwsRegion
	}
	if u.cfg.AwsAccessKeyID != "" && u.cfg.AwsSecretAccessKey != "" {
		awsConfig.Credentials = credentials.NewStaticCredentials(u.cfg.AwsAccessKeyID, u.cfg.AwsSecretAccessKey, "")
	}
	return s3.New(session.New(awsConfig))
}

func (u S3Driver) Save(filePath string, data []byte) error {
	svc := u.connectToS3()

	bucket := u.cfg.UploadPath
	key := filepath.ToSlash(filePath)
	contentType := http.DetectContentType(data)
	_, err := svc.PutObject(&s3.PutObjectInput{
		Body:        bytes.NewReader(data),
		ContentType: &contentType,
		Bucket:      &bucket,
		Key:         &key,
	})
	return err
}

func (u S3Driver) Delete(filePath string) error {
	svc := u.connectToS3()

	bucket := u.cfg.UploadPath
	key := filepath.ToSlash(filePath)

	_, err := svc.DeleteObject(&s3.DeleteObjectInput{
		Bucket: &bucket,
		Key:    &key,
	})
	return err
}
func (u S3Driver) DeleteAll(dirPath string) error {

	svc := u.connectToS3()

	bucket := u.cfg.UploadPath
	prefix := filepath.ToSlash(dirPath)

	listOutput, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
		Bucket: &bucket,
		Prefix: &prefix,
	})
	if err != nil {
		return err
	}
	for _, object := range listOutput.Contents {
		_, err := svc.DeleteObject(&s3.DeleteObjectInput{
			Bucket: &bucket,
			Key:    object.Key,
		})
		if err != nil {
			return err
		}
	}

	return nil
}

func (u S3Driver) List(dirPath string) ([]string, []string, []string, error) {
	var names []string
	var origURLs []string
	var thumbURLs []string

	svc := u.connectToS3()

	bucket := u.cfg.UploadPath
	prefix := filepath.ToSlash(filepath.Join(dirPath, "images"))

	listOutput, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
		Bucket: &bucket,
		Prefix: &prefix,
	})
	if err != nil {
		return names, origURLs, thumbURLs, err
	}

	for _, object := range listOutput.Contents {
		origReq, _ := svc.GetObjectRequest(&s3.GetObjectInput{
			Bucket: &bucket,
			Key:    object.Key,
		})
		origURL, err := origReq.Presign(30 * time.Minute)
		if err == nil {
			urlSegments := strings.Split(*object.Key, "/")
			name := urlSegments[len(urlSegments)-1]

			names = append(names, name)
			origURLs = append(origURLs, origURL)

			thumbKey := strings.Replace(*object.Key, "/images/", "/thumbs/", 1)
			thumbReq, _ := svc.GetObjectRequest(&s3.GetObjectInput{
				Bucket: &bucket,
				Key:    &thumbKey,
			})
			thumbURL, err := thumbReq.Presign(30 * time.Minute)
			if err == nil {
				thumbURLs = append(thumbURLs, thumbURL)
			} else {
				thumbURLs = append(thumbURLs, "")
			}
		}
	}

	return names, origURLs, thumbURLs, nil
}
<|endoftext|>


<commit_before>package server

import (
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
)

type Request interface {
	context.Context

	// Id of this message, used to correlate the response
	Id() string
	// ContentType of the payload
	ContentType() string
	// Payload of raw bytes received from the transport
	Payload() []byte
	// Body is the Unmarshalled `Payload()`. If `RequestType()` is set on
	// the `Endpoint`, we can attempt to unmarshal it for you
	Body() interface{}
	// SetBody of this request
	SetBody(interface{})
	// Service which this request was intended for
	Service() string
	// Endpoint to be called on the receiving service
	Endpoint() string
	// ScopedRequest makes a client request within the scope of the current request
	// @todo change the request & response interface to decouple from protobuf
	ScopedRequest(service string, endpoint string, req proto.Message, resp proto.Message) error

	// AccessToken provided on this request
	AccessToken() string
}
<commit_msg>Switch out access token for credentials<commit_after>package server

import (
	"github.com/b2aio/typhon/auth"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
)

// Request received by the server
type Request interface {
	context.Context

	// Id of this message, used to correlate the response
	Id() string
	// ContentType of the payload
	ContentType() string
	// Payload of raw bytes received from the transport
	Payload() []byte
	// Body is the Unmarshalled `Payload()`. If `RequestType()` is set on
	// the `Endpoint`, we can attempt to unmarshal it for you
	Body() interface{}
	// SetBody of this request
	SetBody(interface{})
	// Service which this request was intended for
	Service() string
	// Endpoint to be called on the receiving service
	Endpoint() string
	// ScopedRequest makes a client request within the scope of the current request
	// @todo change the request & response interface to decouple from protobuf
	ScopedRequest(service string, endpoint string, req proto.Message, resp proto.Message) error

	// Credentials provided on this request
	Credentials() auth.Credentials
	// SetCredentials for this request, useful at api level and for mocking
	SetCredentials() auth.Credentials
}
<|endoftext|>


<commit_before>package testing

import (
	"testing"

	"github.com/jetstack-experimental/navigator/pkg/apis/navigator/v1alpha1"
	"github.com/jetstack-experimental/navigator/pkg/controllers/cassandra"
	"github.com/jetstack-experimental/navigator/pkg/controllers/cassandra/service"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes/fake"
	"k8s.io/client-go/tools/record"
)

func ClusterForTest() *v1alpha1.CassandraCluster {
	c := &v1alpha1.CassandraCluster{}
	c.SetName("cassandra-1")
	c.SetNamespace("app-1")
	return c
}

type Fixture struct {
	t              *testing.T
	Cluster        *v1alpha1.CassandraCluster
	ServiceControl service.Interface
	k8sClient      *fake.Clientset
	k8sObjects     []runtime.Object
}

func NewFixture(t *testing.T) *Fixture {
	return &Fixture{
		t:       t,
		Cluster: ClusterForTest(),
	}
}

func (f *Fixture) AddObjectK(o runtime.Object) {
	f.k8sObjects = append(f.k8sObjects, o)
}

func (f *Fixture) setupAndSync() error {
	recorder := record.NewFakeRecorder(0)
	finished := make(chan struct{})
	defer func() {
		close(recorder.Events)
		<-finished
	}()
	go func() {
		for e := range recorder.Events {
			f.t.Logf("EVENT: %q", e)
		}
		close(finished)
	}()
	f.k8sClient = fake.NewSimpleClientset(f.k8sObjects...)
	if f.ServiceControl == nil {
		k8sFactory := informers.NewSharedInformerFactory(f.k8sClient, 0)
		services := k8sFactory.Core().V1().Services().Lister()
		f.ServiceControl = service.NewControl(f.k8sClient, services, recorder)
	}
	c := cassandra.NewControl(
		f.ServiceControl,
		recorder,
	)
	return c.Sync(f.Cluster)
}

func (f *Fixture) Run() {
	err := f.setupAndSync()
	if err != nil {
		f.t.Error(err)
	}
}

func (f *Fixture) RunExpectError() {
	err := f.setupAndSync()
	if err == nil {
		f.t.Error(err)
	}
}

func (f *Fixture) Services() *v1.ServiceList {
	services, err := f.k8sClient.
		CoreV1().
		Services(f.Cluster.Namespace).
		List(metav1.ListOptions{})
	if err != nil {
		f.t.Fatal(err)
	}
	return services
}

func (f *Fixture) AssertServicesLength(l int) {
	services := f.Services()
	servicesLength := len(services.Items)
	if servicesLength != 1 {
		f.t.Log(services)
		f.t.Errorf(
			"Incorrect number of services: %#v", servicesLength,
		)
	}
}

type FakeControl struct {
	SyncError error
}

func (c *FakeControl) Sync(cluster *v1alpha1.CassandraCluster) error {
	return c.SyncError
}
<commit_msg>Wait for the shared cache to be synced before running the control...so that it can find the supplied objects from the lister.<commit_after>package testing

import (
	"testing"

	"github.com/jetstack-experimental/navigator/pkg/apis/navigator/v1alpha1"
	"github.com/jetstack-experimental/navigator/pkg/controllers/cassandra"
	"github.com/jetstack-experimental/navigator/pkg/controllers/cassandra/service"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes/fake"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
)

func ClusterForTest() *v1alpha1.CassandraCluster {
	c := &v1alpha1.CassandraCluster{}
	c.SetName("cassandra-1")
	c.SetNamespace("app-1")
	return c
}

type Fixture struct {
	t              *testing.T
	Cluster        *v1alpha1.CassandraCluster
	ServiceControl service.Interface
	k8sClient      *fake.Clientset
	k8sObjects     []runtime.Object
}

func NewFixture(t *testing.T) *Fixture {
	return &Fixture{
		t:       t,
		Cluster: ClusterForTest(),
	}
}

func (f *Fixture) AddObjectK(o runtime.Object) {
	f.k8sObjects = append(f.k8sObjects, o)
}

func (f *Fixture) setupAndSync() error {
	recorder := record.NewFakeRecorder(0)
	finished := make(chan struct{})
	defer func() {
		close(recorder.Events)
		<-finished
	}()
	go func() {
		for e := range recorder.Events {
			f.t.Logf("EVENT: %q", e)
		}
		close(finished)
	}()
	f.k8sClient = fake.NewSimpleClientset(f.k8sObjects...)
	k8sFactory := informers.NewSharedInformerFactory(f.k8sClient, 0)
	if f.ServiceControl == nil {
		services := k8sFactory.Core().V1().Services().Lister()
		f.ServiceControl = service.NewControl(f.k8sClient, services, recorder)
	}
	c := cassandra.NewControl(
		f.ServiceControl,
		recorder,
	)
	stopCh := make(chan struct{})
	defer close(stopCh)
	k8sFactory.Start(stopCh)
	if !cache.WaitForCacheSync(
		stopCh,
		k8sFactory.Core().V1().Services().Informer().HasSynced,
	) {
		f.t.Fatal("WaitForCacheSync failure")
	}
	return c.Sync(f.Cluster)
}

func (f *Fixture) Run() {
	err := f.setupAndSync()
	if err != nil {
		f.t.Error(err)
	}
}

func (f *Fixture) RunExpectError() {
	err := f.setupAndSync()
	if err == nil {
		f.t.Error(err)
	}
}

func (f *Fixture) Services() *v1.ServiceList {
	services, err := f.k8sClient.
		CoreV1().
		Services(f.Cluster.Namespace).
		List(metav1.ListOptions{})
	if err != nil {
		f.t.Fatal(err)
	}
	return services
}

func (f *Fixture) AssertServicesLength(l int) {
	services := f.Services()
	servicesLength := len(services.Items)
	if servicesLength != 1 {
		f.t.Log(services)
		f.t.Errorf(
			"Incorrect number of services: %#v", servicesLength,
		)
	}
}

type FakeControl struct {
	SyncError error
}

func (c *FakeControl) Sync(cluster *v1alpha1.CassandraCluster) error {
	return c.SyncError
}
<|endoftext|>


<commit_before>package language

import (
	"strconv"
	"gopkg.in/ldap.v2"
	"fmt"
	"github.com/Frostman/aptomi/pkg/slinga/language/yaml"
	. "github.com/Frostman/aptomi/pkg/slinga/db"
	. "github.com/Frostman/aptomi/pkg/slinga/util"
	. "github.com/Frostman/aptomi/pkg/slinga/log"
	log "github.com/Sirupsen/logrus"
	"github.com/mattn/go-zglob"
	"strings"
)

type LDAPConfig struct {
	Host              string
	Port              int
	BaseDN            string
	Filter            string
	LabelToAtrributes map[string]string
}

// Loads LDAP configuration
func loadLDAPConfig(baseDir string) *LDAPConfig {
	files, _ := zglob.Glob(GetAptomiObjectFilePatternYaml(baseDir, TypeUsersLDAP))
	fileName, err := EnsureSingleFile(files)
	if err != nil {
		Debug.WithFields(log.Fields{
			"error": err,
		}).Panic("LDAP config lookup error")
	}
	result := yaml.LoadObjectFromFile(fileName, &LDAPConfig{}).(*LDAPConfig)

	Debug.WithFields(log.Fields{
		"config": fmt.Sprintf("%v", result),
	}).Info("Loaded LDAP config and mappings")

	return result
}

// Returns the list of attributes to be retrieved from LDAP
func (config *LDAPConfig) getAttributes() []string {
	result := []string{}
	for _, attr := range config.LabelToAtrributes {
		result = append(result, attr)
	}
	return result
}

// UserLoaderFromLDAP allows aptomi to load users from LDAP
type UserLoaderFromLDAP struct {
	config      *LDAPConfig
	cachedUsers *GlobalUsers
}

// NewUserLoaderFromLDAP returns new UserLoaderFromLDAP, given location with LDAP configuration file (with host/port and mapping)
func NewUserLoaderFromLDAP(baseDir string) UserLoader {
	return &UserLoaderFromLDAP{config: loadLDAPConfig(baseDir)}
}

// LoadUsersAll loads all users
func (loader *UserLoaderFromLDAP) LoadUsersAll() GlobalUsers {
	if loader.cachedUsers == nil {
		loader.cachedUsers = &GlobalUsers{Users: make(map[string]*User)}
		t := loader.ldapSearch()
		for _, u := range t {
			// load secrets
			u.Secrets = LoadUserSecretsByIDFromDir(GetAptomiPolicyDir(), u.ID)

			// add user
			loader.cachedUsers.Users[u.ID] = u
		}

	}
	return *loader.cachedUsers
}

// LoadUserByID loads a single user by ID
func (loader *UserLoaderFromLDAP) LoadUserByID(id string) *User {
	return loader.LoadUsersAll().Users[id]
}

// Summary returns summary as string
func (loader *UserLoaderFromLDAP) Summary() string {
	return strconv.Itoa(len(loader.LoadUsersAll().Users)) + " (from LDAP)"
}

// Does search on LDAP and returns entries
func (loader *UserLoaderFromLDAP) ldapSearch() []*User {
	Debug.WithFields(log.Fields{
		"host": loader.config.Host,
		"post": loader.config.Port,
	}).Info("Opening connection to LDAP")

	l, err := ldap.Dial("tcp", fmt.Sprintf("%s:%d", loader.config.Host, loader.config.Port))
	if err != nil {
		panic(err)
	}
	defer l.Close()

	searchRequest := ldap.NewSearchRequest(
		loader.config.BaseDN,
		ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
		loader.config.Filter,
		loader.config.getAttributes(),
		nil,
	)

	Debug.WithFields(log.Fields{
		"baseDN":     loader.config.BaseDN,
		"filter":     loader.config.Filter,
		"attributes": loader.config.getAttributes(),
	}).Info("Making search request to LDAP")

	searchResult, err := l.Search(searchRequest)
	if err != nil {
		panic(err)
	}

	Debug.WithFields(log.Fields{
		"count": len(searchResult.Entries),
	}).Info("Found entries")

	result := []*User{}
	for _, entry := range searchResult.Entries {
		user := &User{
			ID:     entry.DN,
			Name:   entry.GetAttributeValue(loader.config.LabelToAtrributes["name"]),
			Labels: make(map[string]string),
		}
		for label, attr := range loader.config.LabelToAtrributes {
			if label != "id" && label != "name" {
				value := entry.GetAttributeValue(attr)
				if len(value) > 0 {
					user.Labels[label] = ldapValue(value)
				}
			}
		}

		// fmt.Printf("%+v\n", user)
		result = append(result, user)
	}

	return result
}

func ldapValue(value string) string {
	// normalize boolean values
	if strings.ToLower(value) == "true" {
		return "true"
	}
	if strings.ToLower(value) == "false" {
		return "false"
	}
	return value
}
<commit_msg>demo script update<commit_after>package language

import (
	"strconv"
	"gopkg.in/ldap.v2"
	"fmt"
	"github.com/Frostman/aptomi/pkg/slinga/language/yaml"
	. "github.com/Frostman/aptomi/pkg/slinga/db"
	. "github.com/Frostman/aptomi/pkg/slinga/util"
	. "github.com/Frostman/aptomi/pkg/slinga/log"
	log "github.com/Sirupsen/logrus"
	"github.com/mattn/go-zglob"
	"strings"
)

type LDAPConfig struct {
	Host              string
	Port              int
	BaseDN            string
	Filter            string
	LabelToAtrributes map[string]string
}

// Loads LDAP configuration
func loadLDAPConfig(baseDir string) *LDAPConfig {
	files, _ := zglob.Glob(GetAptomiObjectFilePatternYaml(baseDir, TypeUsersLDAP))
	fileName, err := EnsureSingleFile(files)
	if err != nil {
		Debug.WithFields(log.Fields{
			"error": err,
		}).Panic("LDAP config lookup error")
	}
	result := yaml.LoadObjectFromFile(fileName, &LDAPConfig{}).(*LDAPConfig)

	Debug.WithFields(log.Fields{
		"config": fmt.Sprintf("%v", result),
	}).Info("Loaded LDAP config and mappings")

	return result
}

// Returns the list of attributes to be retrieved from LDAP
func (config *LDAPConfig) getAttributes() []string {
	result := []string{}
	for _, attr := range config.LabelToAtrributes {
		result = append(result, attr)
	}
	return result
}

// UserLoaderFromLDAP allows aptomi to load users from LDAP
type UserLoaderFromLDAP struct {
	baseDir     string
	config      *LDAPConfig
	cachedUsers *GlobalUsers
}

// NewUserLoaderFromLDAP returns new UserLoaderFromLDAP, given location with LDAP configuration file (with host/port and mapping)
func NewUserLoaderFromLDAP(baseDir string) UserLoader {
	return &UserLoaderFromLDAP{baseDir: baseDir, config: loadLDAPConfig(baseDir)}
}

// LoadUsersAll loads all users
func (loader *UserLoaderFromLDAP) LoadUsersAll() GlobalUsers {
	if loader.cachedUsers == nil {
		loader.cachedUsers = &GlobalUsers{Users: make(map[string]*User)}
		t := loader.ldapSearch()
		for _, u := range t {
			// load secrets
			u.Secrets = LoadUserSecretsByIDFromDir(loader.baseDir, u.ID)

			// add user
			loader.cachedUsers.Users[u.ID] = u
		}

	}
	return *loader.cachedUsers
}

// LoadUserByID loads a single user by ID
func (loader *UserLoaderFromLDAP) LoadUserByID(id string) *User {
	return loader.LoadUsersAll().Users[id]
}

// Summary returns summary as string
func (loader *UserLoaderFromLDAP) Summary() string {
	return strconv.Itoa(len(loader.LoadUsersAll().Users)) + " (from LDAP)"
}

// Does search on LDAP and returns entries
func (loader *UserLoaderFromLDAP) ldapSearch() []*User {
	Debug.WithFields(log.Fields{
		"host": loader.config.Host,
		"post": loader.config.Port,
	}).Info("Opening connection to LDAP")

	l, err := ldap.Dial("tcp", fmt.Sprintf("%s:%d", loader.config.Host, loader.config.Port))
	if err != nil {
		panic(err)
	}
	defer l.Close()

	searchRequest := ldap.NewSearchRequest(
		loader.config.BaseDN,
		ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
		loader.config.Filter,
		loader.config.getAttributes(),
		nil,
	)

	Debug.WithFields(log.Fields{
		"baseDN":     loader.config.BaseDN,
		"filter":     loader.config.Filter,
		"attributes": loader.config.getAttributes(),
	}).Info("Making search request to LDAP")

	searchResult, err := l.Search(searchRequest)
	if err != nil {
		panic(err)
	}

	Debug.WithFields(log.Fields{
		"count": len(searchResult.Entries),
	}).Info("Found entries")

	result := []*User{}
	for _, entry := range searchResult.Entries {
		user := &User{
			ID:     entry.DN,
			Name:   entry.GetAttributeValue(loader.config.LabelToAtrributes["name"]),
			Labels: make(map[string]string),
		}
		for label, attr := range loader.config.LabelToAtrributes {
			if label != "id" && label != "name" {
				value := entry.GetAttributeValue(attr)
				if len(value) > 0 {
					user.Labels[label] = ldapValue(value)
				}
			}
		}

		// fmt.Printf("%+v\n", user)
		result = append(result, user)
	}

	return result
}

func ldapValue(value string) string {
	// normalize boolean values
	if strings.ToLower(value) == "true" {
		return "true"
	}
	if strings.ToLower(value) == "false" {
		return "false"
	}
	return value
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

import (
	"fmt"
	"testing"

	"k8s.io/kubernetes/pkg/util/exec"
)

type ErrorMounter struct {
	*FakeMounter
	errIndex int
	err      []error
}

func (mounter *ErrorMounter) Mount(source string, target string, fstype string, options []string) error {
	i := mounter.errIndex
	mounter.errIndex++
	if mounter.err != nil && mounter.err[i] != nil {
		return mounter.err[i]
	}
	return mounter.FakeMounter.Mount(source, target, fstype, options)
}

type ExecArgs struct {
	command string
	args    []string
	output  string
	err     error
}

func TestSafeFormatAndMount(t *testing.T) {
	tests := []struct {
		fstype        string
		mountOptions  []string
		execScripts   []ExecArgs
		mountErrs     []error
		expectedError error
	}{
		{ // Test a read only mount
			fstype:       "ext4",
			mountOptions: []string{"ro"},
		},
		{ // Test a normal mount
			fstype: "ext4",
		},

		{ // Test that 'lsblk' is called and fails
			fstype:    "ext4",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'")},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "ext4", nil},
			},
			expectedError: fmt.Errorf("unknown filesystem type '(null)'"),
		},
		{ // Test that 'lsblk' is called and confirms unformatted disk, format fails
			fstype:    "ext4",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'")},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "", nil},
				{"mkfs.ext4", []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", "/dev/foo"}, "", fmt.Errorf("formatting failed")},
			},
			expectedError: fmt.Errorf("formatting failed"),
		},
		{ // Test that 'lsblk' is called and confirms unformatted disk, format passes, second mount fails
			fstype:    "ext4",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'"), fmt.Errorf("Still cannot mount")},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "", nil},
				{"mkfs.ext4", []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", "/dev/foo"}, "", nil},
			},
			expectedError: fmt.Errorf("Still cannot mount"),
		},
		{ // Test that 'lsblk' is called and confirms unformatted disk, format passes, second mount passes
			fstype:    "ext4",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'"), nil},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "", nil},
				{"mkfs.ext4", []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", "/dev/foo"}, "", nil},
			},
			expectedError: nil,
		},
		{ // Test that 'lsblk' is called and confirms unformatted disk, format passes, second mount passes with ext3
			fstype:    "ext3",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'"), nil},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "", nil},
				{"mkfs.ext3", []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", "/dev/foo"}, "", nil},
			},
			expectedError: nil,
		},
		{ // Test that 'lsblk' is called and confirms unformatted disk, format passes, second mount passes
			// test that none ext4 fs does not get called with ext4 options.
			fstype:    "xfs",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'"), nil},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "", nil},
				{"mkfs.xfs", []string{"/dev/foo"}, "", nil},
			},
			expectedError: nil,
		},
	}

	for _, test := range tests {
		commandScripts := []exec.FakeCommandAction{}
		for _, expected := range test.execScripts {
			ecmd := expected.command
			eargs := expected.args
			output := expected.output
			err := expected.err
			commandScript := func(cmd string, args ...string) exec.Cmd {
				if cmd != ecmd {
					t.Errorf("Unexpected command %s. Expecting %s", cmd, ecmd)
				}

				for j := range args {
					if args[j] != eargs[j] {
						t.Errorf("Unexpected args %v. Expecting %v", args, eargs)
					}
				}
				fake := exec.FakeCmd{
					CombinedOutputScript: []exec.FakeCombinedOutputAction{
						func() ([]byte, error) { return []byte(output), err },
					},
				}
				return exec.InitFakeCmd(&fake, cmd, args...)
			}
			commandScripts = append(commandScripts, commandScript)
		}

		fake := exec.FakeExec{
			CommandScript: commandScripts,
		}

		fakeMounter := ErrorMounter{&FakeMounter{}, 0, test.mountErrs}
		mounter := SafeFormatAndMount{
			Interface: &fakeMounter,
			Runner:    &fake,
		}

		device := "/dev/foo"
		dest := "/mnt/bar"
		err := mounter.FormatAndMount(device, dest, test.fstype, test.mountOptions)
		if test.expectedError == nil {
			if err != nil {
				t.Errorf("unexpected non-error: %v", err)
			}

			// Check that something was mounted on the directory
			isNotMountPoint, err := fakeMounter.IsLikelyNotMountPoint(dest)
			if err != nil || isNotMountPoint {
				t.Errorf("the directory was not mounted")
			}

			//check that the correct device was mounted
			mountedDevice, _, err := GetDeviceNameFromMount(fakeMounter.FakeMounter, dest)
			if err != nil || mountedDevice != device {
				t.Errorf("the correct device was not mounted")
			}
		} else {
			if err == nil || test.expectedError.Error() != err.Error() {
				t.Errorf("unexpected error: %v. Expecting %v", err, test.expectedError)
			}
		}
	}
}
<commit_msg>Skip format and mount test on windows or mac<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

import (
	"fmt"
	"runtime"
	"testing"

	"k8s.io/kubernetes/pkg/util/exec"
)

type ErrorMounter struct {
	*FakeMounter
	errIndex int
	err      []error
}

func (mounter *ErrorMounter) Mount(source string, target string, fstype string, options []string) error {
	i := mounter.errIndex
	mounter.errIndex++
	if mounter.err != nil && mounter.err[i] != nil {
		return mounter.err[i]
	}
	return mounter.FakeMounter.Mount(source, target, fstype, options)
}

type ExecArgs struct {
	command string
	args    []string
	output  string
	err     error
}

func TestSafeFormatAndMount(t *testing.T) {
	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
		t.Skipf("not supported on GOOS=%s", runtime.GOOS)
	}
	tests := []struct {
		fstype        string
		mountOptions  []string
		execScripts   []ExecArgs
		mountErrs     []error
		expectedError error
	}{
		{ // Test a read only mount
			fstype:       "ext4",
			mountOptions: []string{"ro"},
		},
		{ // Test a normal mount
			fstype: "ext4",
		},

		{ // Test that 'lsblk' is called and fails
			fstype:    "ext4",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'")},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "ext4", nil},
			},
			expectedError: fmt.Errorf("unknown filesystem type '(null)'"),
		},
		{ // Test that 'lsblk' is called and confirms unformatted disk, format fails
			fstype:    "ext4",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'")},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "", nil},
				{"mkfs.ext4", []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", "/dev/foo"}, "", fmt.Errorf("formatting failed")},
			},
			expectedError: fmt.Errorf("formatting failed"),
		},
		{ // Test that 'lsblk' is called and confirms unformatted disk, format passes, second mount fails
			fstype:    "ext4",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'"), fmt.Errorf("Still cannot mount")},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "", nil},
				{"mkfs.ext4", []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", "/dev/foo"}, "", nil},
			},
			expectedError: fmt.Errorf("Still cannot mount"),
		},
		{ // Test that 'lsblk' is called and confirms unformatted disk, format passes, second mount passes
			fstype:    "ext4",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'"), nil},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "", nil},
				{"mkfs.ext4", []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", "/dev/foo"}, "", nil},
			},
			expectedError: nil,
		},
		{ // Test that 'lsblk' is called and confirms unformatted disk, format passes, second mount passes with ext3
			fstype:    "ext3",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'"), nil},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "", nil},
				{"mkfs.ext3", []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", "/dev/foo"}, "", nil},
			},
			expectedError: nil,
		},
		{ // Test that 'lsblk' is called and confirms unformatted disk, format passes, second mount passes
			// test that none ext4 fs does not get called with ext4 options.
			fstype:    "xfs",
			mountErrs: []error{fmt.Errorf("unknown filesystem type '(null)'"), nil},
			execScripts: []ExecArgs{
				{"lsblk", []string{"-nd", "-o", "FSTYPE", "/dev/foo"}, "", nil},
				{"mkfs.xfs", []string{"/dev/foo"}, "", nil},
			},
			expectedError: nil,
		},
	}

	for _, test := range tests {
		commandScripts := []exec.FakeCommandAction{}
		for _, expected := range test.execScripts {
			ecmd := expected.command
			eargs := expected.args
			output := expected.output
			err := expected.err
			commandScript := func(cmd string, args ...string) exec.Cmd {
				if cmd != ecmd {
					t.Errorf("Unexpected command %s. Expecting %s", cmd, ecmd)
				}

				for j := range args {
					if args[j] != eargs[j] {
						t.Errorf("Unexpected args %v. Expecting %v", args, eargs)
					}
				}
				fake := exec.FakeCmd{
					CombinedOutputScript: []exec.FakeCombinedOutputAction{
						func() ([]byte, error) { return []byte(output), err },
					},
				}
				return exec.InitFakeCmd(&fake, cmd, args...)
			}
			commandScripts = append(commandScripts, commandScript)
		}

		fake := exec.FakeExec{
			CommandScript: commandScripts,
		}

		fakeMounter := ErrorMounter{&FakeMounter{}, 0, test.mountErrs}
		mounter := SafeFormatAndMount{
			Interface: &fakeMounter,
			Runner:    &fake,
		}

		device := "/dev/foo"
		dest := "/mnt/bar"
		err := mounter.FormatAndMount(device, dest, test.fstype, test.mountOptions)
		if test.expectedError == nil {
			if err != nil {
				t.Errorf("unexpected non-error: %v", err)
			}

			// Check that something was mounted on the directory
			isNotMountPoint, err := fakeMounter.IsLikelyNotMountPoint(dest)
			if err != nil || isNotMountPoint {
				t.Errorf("the directory was not mounted")
			}

			//check that the correct device was mounted
			mountedDevice, _, err := GetDeviceNameFromMount(fakeMounter.FakeMounter, dest)
			if err != nil || mountedDevice != device {
				t.Errorf("the correct device was not mounted")
			}
		} else {
			if err == nil || test.expectedError.Error() != err.Error() {
				t.Errorf("unexpected error: %v. Expecting %v", err, test.expectedError)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 The Protogalaxy Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package service

import (
	"github.com/golang/glog"
	"github.com/protogalaxy/service-goroom/lobby"
)

type MissingParameterError string

func NewMissingParameterError(name string) MissingParameterError {
	return MissingParameterError(name)
}

func (e MissingParameterError) Error() string {
	return "Missing parameter: " + string(e)
}

const (
	RoomStatusCreated       = "created"
	RoomStatusAlreadyInRoom = "already_in_room"
	RoomStatusFull          = "room_full"
	RoomStatusFound         = "found"
	RoomStatusNotFound      = "not_found"
	RoomStatusJoined        = "joined"
)

type GoRoom struct {
	Lobby lobby.Lobby
}

type CreateRoomRequest struct {
	UserID string `json:"user_id"`
}

type CreateRoomResponse struct {
	Status string `json:"status"`
	RoomID string `json:"room_id"`
}

func (s *GoRoom) CreateRoom(req *CreateRoomRequest, res *CreateRoomResponse) error {
	if req.UserID == "" {
		return NewMissingParameterError("user id")
	}

	rid, err := s.Lobby.CreateRoom(req.UserID)
	if err == lobby.ErrAlreadyInRoom {
		res.Status = RoomStatusAlreadyInRoom
		return nil
	} else if err != nil {
		glog.Errorf("Unexpected error: %s", err)
		return err
	}
	res.Status = RoomStatusCreated
	res.RoomID = rid
	return nil
}

type JoinRoomRequest struct {
	UserID string `json:"user_id"`
	RoomID string `json:"-"`
}

type JoinRoomResponse struct {
	Status string `json:"status"`
}

func (s *GoRoom) JoinRoom(req *JoinRoomRequest, res *JoinRoomResponse) error {
	if req.UserID == "" {
		return NewMissingParameterError("user id")
	} else if req.RoomID == "" {
		return NewMissingParameterError("room id")
	}

	err := s.Lobby.JoinRoom(req.RoomID, req.UserID)

	if err == lobby.ErrRoomNotFound {
		res.Status = RoomStatusNotFound
	} else if err == lobby.ErrAlreadyInRoom {
		res.Status = RoomStatusAlreadyInRoom
	} else if err == lobby.ErrRoomFull {
		res.Status = RoomStatusFull
	} else if err != nil {
		glog.Errorf("Unexpected error: %s", err)
		return err
	} else {
		res.Status = RoomStatusJoined
	}
	return nil
}

type RoomInfoRequest struct {
	RoomID string `json:"-"`
}

type Room struct {
	RoomID      string `json:"room_id"`
	Owner       string `json:"owner"`
	OtherPlayer string `json:"other_player,omitempty"`
}

type RoomInfoResponse struct {
	Status string `json:"status"`
	Room   *Room  `json:"room"`
}

func (s *GoRoom) RoomInfo(req *RoomInfoRequest, res *RoomInfoResponse) error {
	room, err := s.Lobby.RoomInfo(req.RoomID)
	if err == lobby.ErrRoomNotFound {
		res.Status = RoomStatusNotFound
	} else if err != nil {
		glog.Errorf("Unexpected error: %s", err)
		return err
	} else {
		res.Status = RoomStatusFound
		res.Room = &Room{
			RoomID:      room.ID,
			Owner:       room.Owner,
			OtherPlayer: room.OtherPlayer,
		}
	}
	return nil
}
<commit_msg>Refactored common error to status handling.<commit_after>// Copyright (C) 2015 The Protogalaxy Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package service

import (
	"github.com/golang/glog"
	"github.com/protogalaxy/service-goroom/lobby"
)

type MissingParameterError string

func NewMissingParameterError(name string) MissingParameterError {
	return MissingParameterError(name)
}

func (e MissingParameterError) Error() string {
	return "Missing parameter: " + string(e)
}

const (
	RoomStatusCreated       = "created"
	RoomStatusAlreadyInRoom = "already_in_room"
	RoomStatusFull          = "room_full"
	RoomStatusFound         = "found"
	RoomStatusNotFound      = "not_found"
	RoomStatusJoined        = "joined"
)

type GoRoom struct {
	Lobby lobby.Lobby
}

type CreateRoomRequest struct {
	UserID string `json:"user_id"`
}

type CreateRoomResponse struct {
	Status string `json:"status"`
	RoomID string `json:"room_id"`
}

func (s *GoRoom) CreateRoom(req *CreateRoomRequest, res *CreateRoomResponse) error {
	if req.UserID == "" {
		return NewMissingParameterError("user id")
	}

	rid, err := s.Lobby.CreateRoom(req.UserID)
	if err != nil {
		return errorToStatus(&res.Status, err)
	}

	res.Status = RoomStatusCreated
	res.RoomID = rid
	return nil
}

type JoinRoomRequest struct {
	UserID string `json:"user_id"`
	RoomID string `json:"-"`
}

type JoinRoomResponse struct {
	Status string `json:"status"`
}

func (s *GoRoom) JoinRoom(req *JoinRoomRequest, res *JoinRoomResponse) error {
	if req.UserID == "" {
		return NewMissingParameterError("user id")
	} else if req.RoomID == "" {
		return NewMissingParameterError("room id")
	}

	err := s.Lobby.JoinRoom(req.RoomID, req.UserID)
	if err != nil {
		return errorToStatus(&res.Status, err)
	}

	res.Status = RoomStatusJoined
	return nil
}

type RoomInfoRequest struct {
	RoomID string `json:"-"`
}

type Room struct {
	RoomID      string `json:"room_id"`
	Owner       string `json:"owner"`
	OtherPlayer string `json:"other_player,omitempty"`
}

type RoomInfoResponse struct {
	Status string `json:"status"`
	Room   *Room  `json:"room"`
}

func (s *GoRoom) RoomInfo(req *RoomInfoRequest, res *RoomInfoResponse) error {
	room, err := s.Lobby.RoomInfo(req.RoomID)
	if err != nil {
		return errorToStatus(&res.Status, err)
	}
	res.Status = RoomStatusFound
	res.Room = &Room{
		RoomID:      room.ID,
		Owner:       room.Owner,
		OtherPlayer: room.OtherPlayer,
	}
	return nil
}

func errorToStatus(s *string, err error) error {
	if err == lobby.ErrRoomNotFound {
		*s = RoomStatusNotFound
	} else if err == lobby.ErrAlreadyInRoom {
		*s = RoomStatusAlreadyInRoom
	} else if err == lobby.ErrRoomFull {
		*s = RoomStatusFull
	} else if err != nil {
		glog.Errorf("Unexpected error: %s", err)
		return err
	} else {
		glog.Fatal("Error should not be nil")
	}
	return nil
}
<|endoftext|>


<commit_before>package service

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/aosfather/bingo"
	"github.com/aosfather/bingo/utils"
	"github.com/go-redis/redis"
)

/**
  搜索实现
  通过倒排实现关键信息的实现
  规则：
   1、原始内容使用hashmap存储，对象【ID，Content】，键值 indexname，二级key根据md5 对象转json字符串
   2、针对原始内容带的标签，key value，生成set，名称为 indexname_key_value的形式，set内容放 通过内容md5出来的键值
   3、搜索的时候，根据传递的搜索条件 key，value 数组，对找到的set（形如indexname_key_value ），实行找交集
   4、根据交集的结果二级key，并从indexname的hashmap中获取json内容

*/

type Field struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type PageSearchResult struct {
	Id    string `json:"uuid"` //查询的请求id
	Index int64  `json:"page"` //页码
	Data  []TargetObject
}

type TargetObject struct {
	Id   string          `json:"id"`
	Data json.RawMessage `json:"data"`
}
type SourceObject struct {
	TargetObject
	Fields map[string][]string `json:"fields"`
}

type FieldType byte
const (
	FT_TEXT FieldType =11  //文本类型
	FT_NUMBER FieldType =9 //数字
	FT_ENUM   FieldType =8 //枚举
	FT_ID FieldType =7  //id唯一标识
	FT_DATE FieldType =6 //日期
)
//索引的元数据信息
type IndexMeta struct {
	Name string  //索引名称
	Fields []FieldMeta //字段
}

type FieldMeta struct {
	Name string //字段名称
	Type FieldType //类型

}

type SearchEngine struct {
	indexs   map[string]*searchIndex
	safeIndexs map[string]*searchIndex  //安全flush时候的临时索引
	client   *redis.Client
	logger   utils.Log
	pageSize int64
	pageLife int64 //分钟
}

func (this *SearchEngine) Init(context *bingo.ApplicationContext) {
	fmt.Println("init .....")
	db, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.db"))
	if err != nil {
		db = 0
	}

	size, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagesize"))
	if err != nil {
		size = 20 //默认大小20条
	}
	this.pageSize = int64(size)

	life, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagelife"))
	if err != nil {
		life = 10 //默认时间10分钟
	}
	this.pageLife = int64(life)

	this.client = redis.NewClient(&redis.Options{
		Addr:     context.GetPropertyFromConfig("service.search.redis"),
		Password: "", // no password set
		DB:       db,
	})
	fmt.Println(context.GetPropertyFromConfig("service.search.redis"))
	this.indexs = make(map[string]*searchIndex)
	this.safeIndexs = make(map[string]*searchIndex)
	this.logger = context.GetLog("bingo_search")
}

//创建索引
func (this *SearchEngine) CreateIndex(name string) *searchIndex {
	if name != "" {
		index := this.indexs[name]
		if index == nil {
			index = &searchIndex{name, this}
			this.indexs[name] = index
		}
		return index
	}

	return nil
}

//清除索引，将整个索引的数据摧毁
func (this *SearchEngine)FlushIndex(name string) {
	//1、删除存放的数据
	this.client.Del(name)

	//2、删除所有的索引key
	keys,err:=this.client.Keys(name+"_*").Result()
	if err!=nil {
		this.logger.Debug("get index keys error:%s",err.Error())
	}

	if keys!=nil && len(keys)>0 {
		this.client.Del(keys...)
	}
}

//开始安全flush
func (this *SearchEngine)BeginSafeFlush(name string) {
	index := this.indexs[name]
	if index!=nil {
		var nindexName string
		if strings.HasSuffix(index.name,"#") {
			nindexName=index.name[0:len(index.name)-2]
		}else {
			nindexName=index.name+"#"
		}

		nindex:=&searchIndex{nindexName, this}
		this.safeIndexs[name]=nindex
	}
}

//结束安全flush，自动替换掉查询用的索引
func (this *SearchEngine)EndSafeFlush(name string) {
	index:=this.safeIndexs[name]
	if index!=nil {
		delete(this.safeIndexs,name)
		oldindex:=this.indexs[name]
		this.indexs[name]=index
        //删除旧的索引数据
		if oldindex!=nil {
			this.FlushIndex(oldindex.name)
		}
	}
}

//加载和刷新数据
func (this *SearchEngine) LoadSource(name string, obj *SourceObject) {
	if name==""||obj==nil {
		return
	}

	//判断是否正在进行安全flush，如果是安全flush，所有的更改只发生在正在替换的索引上
	index:=this.safeIndexs[name]
	if index==nil {
		index = this.CreateIndex(name)
	}

	if index != nil {
		index.LoadObject(obj)
	}

}

//删除数据
func (this *SearchEngine)RemoveSource(name string, obj *SourceObject){
   if name!=""&&obj!=nil{
	   index := this.CreateIndex(name)
	   if index != nil {
		   index.RemoveObject(obj)
	   }
   }
}

//删除索引中的某个词条
func (this *SearchEngine)RemoveKeyword(name,field,word string)  {
	if name!="" {
		index := this.CreateIndex(name)
		if index != nil {
			index.RemoveKeyword(field,word)
		}
	}

}

//按页获取数据
func (this *SearchEngine) FetchByPage(request string, page int64) *PageSearchResult {
	if request != "" {
		//获取request的name
		index := strings.Index(request, ":")
		if index < 0 {
			this.logger.Error("pagerequest's index name not found !")
			return nil //找不到对应的索引类型
		}

		name := request[0:index]
		return this.fetchByPage(name,request,page)
	}

	return nil
}

func (this *SearchEngine)fetchByPage(name string,request string,page int64)*PageSearchResult {
	if page <= 0 {
		page = 1
	}
	startIndex := (page - 1) * this.pageSize //+ 1 //从1开始计数
	endIndex := page*this.pageSize - 1
	this.logger.Debug(request,startIndex,endIndex)
	keys, err := this.client.LRange(request, startIndex, endIndex).Result()

	if err != nil || len(keys) == 0 {
		if err!=nil {
			this.logger.Debug(err.Error())
		}
		this.logger.Debug("no content by page!")
		return nil
	}
	go this.client.Expire(request, time.Duration(this.pageLife)*time.Minute) //更新重置失效时间
	return &PageSearchResult{request, page, this.fetch(name, keys...)}
}

//获取内容
func (this *SearchEngine) fetch(name string, keys ...string) []TargetObject {
	datas, err1 := this.client.HMGet(name, keys...).Result()
	if err1 == nil && len(datas) > 0 {

		var targets []TargetObject

		for _, v := range datas {
			if v != nil {
				t := TargetObject{}
				json.Unmarshal([]byte(fmt.Sprintf("%v", v)), &t)
				targets = append(targets, t)
			}
		}

		return targets

	} else {
		this.logger.Error("get data by index error!%s", err1.Error())
	}

	return nil
}

func (this *SearchEngine) Search(name string, input ...Field) *PageSearchResult {
	if name != "" {
		index := this.indexs[name]
		if index != nil {
			return index.Search(input...)

		}
		this.logger.Info("not found index %s", name)
	}

	return nil
}

type searchIndex struct {
	name   string
	engine *SearchEngine
}

//搜索信息
func (this *searchIndex) Search(input ...Field) *PageSearchResult {
	//生成索引搜索请求号
	requestkey := getSearchRequestUuid(this.name)

	//搜索索引
	var searchkeys []string
	var tmpkeys []string
	var notTmpkeys[]string
	for _, f := range input {

		//处理not的情况
		if strings.HasPrefix(f.Key,"!") {//字段使用！表示"非"的意思，就是不等于
		    k:=f.Key[1:len(f.Key)]
			v := f.Value
			arrays := strings.Split(v, "|")
			if len(arrays) > 1 {
				for _, subvalue := range arrays {
					notTmpkeys = append(notTmpkeys, this.buildTheKeyByItem(k, subvalue))
				}
			}else {
				notTmpkeys=append(notTmpkeys,this.buildTheKeyByItem(k, v))
			}

			continue
		}

		//处理并集
		v := f.Value
		arrays := strings.Split(v, "|")  //使用 | 分割多个取值
		if len(arrays) > 1 {
			skey := requestkey + ":" + f.Key
			var subkeys []string
			for _, subkey := range arrays {
				subkeys = append(subkeys, this.buildTheKeyByItem(f.Key, subkey))
			}

			//取并集，将结果存入临时的key中，用于后续取交集用
			this.engine.client.SUnionStore(skey, subkeys...)
			tmpkeys = append(tmpkeys, skey) //放入临时组中用于使用后删除
			searchkeys = append(searchkeys, skey)
		} else {
			searchkeys = append(searchkeys, this.buildTheKey(f))
		}

	}
	defer this.deleteTempkeys(tmpkeys) //删除临时创建的key

	var values []string
	//有差集
	if notTmpkeys!=nil && len(notTmpkeys) >1 {
		interTmpkey:=requestkey+"_tmp"
		//取交集
		result := this.engine.client.SInterStore(interTmpkey,searchkeys...)
		_, err := result.Result()
		if err != nil {
			this.engine.logger.Error("inter key error!%s", err.Error())
			return  nil
		}
		//取差集合，存储
		diffkeys:=make([]string,len(notTmpkeys)+1)
		diffkeys[0]=interTmpkey
		copy(diffkeys[1:len(diffkeys)],notTmpkeys)

		defer this.deleteTempkeys([]string{interTmpkey})//删除临时key
		data:=this.engine.client.SDiff(diffkeys...)
		values, _ = data.Result()

	} else {//如果无差集合，则直接返回交集
		result:=this.engine.client.SInter(searchkeys...)
		values, _ = result.Result()
	}

	//存储数据
	var datas []interface{}

	for _, v := range values {
		datas = append(datas, v)
	}
	this.engine.client.RPush(requestkey,datas...)

	this.engine.client.Expire(requestkey, time.Duration(this.engine.pageLife)*time.Minute) //指定x分钟后失效
	//取出第一页的数据返回
	return this.engine.fetchByPage(this.name,requestkey,1)

}

//删除某个字段等于某值得列表，也就是删除一个词条
func (this *searchIndex)RemoveKeyword(field,word string){
	if field!=""&&word!="" {
		thekey:=this.buildTheKeyByItem(field,word)
		this.engine.client.Del(thekey)
	}

}

//删除临时创建的key
func (this *searchIndex) deleteTempkeys(keys []string) {
	if keys != nil && len(keys) > 0 {
		this.engine.client.Del(keys...)
	}
}

//刷新索引，加载信息到存储中
func (this *searchIndex) LoadObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key:=obj.Id
	//如果没有指明对象id，则使用md5作为数据的唯一标识
	if key=="" {
		key= getMd5str(string(data))
	}

	//1、放入数据到目标集合中
	this.engine.client.HSet(this.name, key, string(data))

	//2、根据field存储到各个对应的索引中
	for k, v := range obj.Fields {
		for _,vkey:=range v{
			this.engine.client.SAdd(this.buildTheKeyByItem(k, vkey), key)
		}

	}

}

//删除数据：删除数据及索引存的值
func (this *searchIndex)RemoveObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key:=obj.Id
	if key=="" {
		key=getMd5str(string(data))
	}
	//删除数据
	this.engine.client.HDel(this.name,key)
	//删除索引里的记录
	for k,v:=range obj.Fields {
		for _,vkey:=range v {
			this.engine.client.SRem(this.buildTheKeyByItem(k, vkey), key)
		}
	}

}

func (this *searchIndex) buildTheKey(f Field) string {
	return this.buildTheKeyByItem(f.Key, f.Value)
}

func (this *searchIndex) buildTheKeyByItem(key, value string) string {
	return fmt.Sprintf("%s_%s_%s", this.name, key, value)
}

func getMd5str(value string) string {
	data := []byte(value)
	has := md5.Sum(data)
	md5str1 := fmt.Sprintf("%x", has) //将[]byte转成16进制

	return md5str1

}

func getSearchRequestUuid(prefix string) string {
	return fmt.Sprintf("%s:%d", prefix, time.Now().UnixNano())
}
<commit_msg>增加索引删除某个词条功能<commit_after>package service

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/aosfather/bingo"
	"github.com/aosfather/bingo/utils"
	"github.com/go-redis/redis"
)

/**
  搜索实现
  通过倒排实现关键信息的实现
  规则：
   1、原始内容使用hashmap存储，对象【ID，Content】，键值 indexname，二级key根据md5 对象转json字符串
   2、针对原始内容带的标签，key value，生成set，名称为 indexname_key_value的形式，set内容放 通过内容md5出来的键值
   3、搜索的时候，根据传递的搜索条件 key，value 数组，对找到的set（形如indexname_key_value ），实行找交集
   4、根据交集的结果二级key，并从indexname的hashmap中获取json内容

*/

type Field struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type PageSearchResult struct {
	Id    string `json:"uuid"` //查询的请求id
	Index int64  `json:"page"` //页码
	Data  []TargetObject
}

type TargetObject struct {
	Id   string          `json:"id"`
	Data json.RawMessage `json:"data"`
}
type SourceObject struct {
	TargetObject
	Fields map[string][]string `json:"fields"`
}

type FieldType byte

const (
	FT_TEXT   FieldType = 11 //文本类型
	FT_NUMBER FieldType = 9  //数字
	FT_ENUM   FieldType = 8  //枚举
	FT_ID     FieldType = 7  //id唯一标识
	FT_DATE   FieldType = 6  //日期
)

//索引的元数据信息
type IndexMeta struct {
	Name   string      //索引名称
	Fields []FieldMeta //字段
}

type FieldMeta struct {
	Name string    //字段名称
	Type FieldType //类型
}

type SearchEngine struct {
	indexs     map[string]*searchIndex
	safeIndexs map[string]*searchIndex //安全flush时候的临时索引
	client     *redis.Client
	logger     utils.Log
	pageSize   int64
	pageLife   int64 //分钟
}

func (this *SearchEngine) Init(context *bingo.ApplicationContext) {
	fmt.Println("init .....")
	db, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.db"))
	if err != nil {
		db = 0
	}

	size, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagesize"))
	if err != nil {
		size = 20 //默认大小20条
	}
	this.pageSize = int64(size)

	life, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagelife"))
	if err != nil {
		life = 10 //默认时间10分钟
	}
	this.pageLife = int64(life)

	this.client = redis.NewClient(&redis.Options{
		Addr:     context.GetPropertyFromConfig("service.search.redis"),
		Password: "", // no password set
		DB:       db,
	})
	fmt.Println(context.GetPropertyFromConfig("service.search.redis"))
	this.indexs = make(map[string]*searchIndex)
	this.safeIndexs = make(map[string]*searchIndex)
	this.logger = context.GetLog("bingo_search")
}

//创建索引
func (this *SearchEngine) CreateIndex(name string) *searchIndex {
	if name != "" {
		index := this.indexs[name]
		if index == nil {
			index = &searchIndex{name, this}
			this.indexs[name] = index
		}
		return index
	}

	return nil
}

//清除索引，将整个索引的数据摧毁
func (this *SearchEngine) FlushIndex(name string) {
	//1、删除存放的数据
	this.client.Del(name)

	//2、删除所有的索引key
	keys, err := this.client.Keys(name + "_*").Result()
	if err != nil {
		this.logger.Debug("get index keys error:%s", err.Error())
	}

	if keys != nil && len(keys) > 0 {
		this.client.Del(keys...)
	}
}

//开始安全flush
func (this *SearchEngine) BeginSafeFlush(name string) {
	index := this.indexs[name]
	if index != nil {
		var nindexName string
		if strings.HasSuffix(index.name, "#") {
			nindexName = index.name[0:len(index.name)-2]
		} else {
			nindexName = index.name + "#"
		}

		nindex := &searchIndex{nindexName, this}
		this.safeIndexs[name] = nindex
	}
}

//结束安全flush，自动替换掉查询用的索引
func (this *SearchEngine) EndSafeFlush(name string) {
	index := this.safeIndexs[name]
	if index != nil {
		delete(this.safeIndexs, name)
		oldindex := this.indexs[name]
		this.indexs[name] = index
		//删除旧的索引数据
		if oldindex != nil {
			this.FlushIndex(oldindex.name)
		}
	}
}

//加载和刷新数据
func (this *SearchEngine) LoadSource(name string, obj *SourceObject) {
	if name == "" || obj == nil {
		return
	}

	//判断是否正在进行安全flush，如果是安全flush，所有的更改只发生在正在替换的索引上
	index := this.safeIndexs[name]
	if index == nil {
		index = this.CreateIndex(name)
	}

	if index != nil {
		index.LoadObject(obj)
	}

}

//删除数据
func (this *SearchEngine) RemoveSource(name string, obj *SourceObject) {
	if name != "" && obj != nil {
		index := this.CreateIndex(name)
		if index != nil {
			index.RemoveObject(obj)
		}
	}
}

//根据查询条件删除数据,只支持单个条件很正向条件的索引字段删除，数据删除支持搜索删除
func (this *SearchEngine) RemoveBySearch(name string, input ...Field) {
	if name != "" && len(input) > 0 {
		index := this.CreateIndex(name)
		if index != nil {
			request := index.Search(input...)
			ids, err := this.client.LRange(request, 0, -1).Result()
			if err != nil {
				this.logger.Debug("remove error %s", err.Error())
			}
			//删除索引词中的数据
			for _, in := range input {
				index.RemoveFromFieldById(in.Key, in.Value, ids...)
			}
			//删除数据
			this.client.HDel(index.name, ids...)
		}

	}
}

//删除索引中的某个词条
func (this *SearchEngine) RemoveKeyword(name, field, word string) {
	if name != "" {
		index := this.CreateIndex(name)
		if index != nil {
			index.RemoveKeyword(field, word)
		}
	}

}

//按页获取数据
func (this *SearchEngine) FetchByPage(request string, page int64) *PageSearchResult {
	if request != "" {
		//获取request的name
		index := strings.Index(request, ":")
		if index < 0 {
			this.logger.Error("pagerequest's index name not found !")
			return nil //找不到对应的索引类型
		}

		name := request[0:index]
		return this.fetchByPage(name, request, page)
	}

	return nil
}

func (this *SearchEngine) fetchByPage(name string, request string, page int64) *PageSearchResult {
	if page <= 0 {
		page = 1
	}
	startIndex := (page - 1) * this.pageSize //+ 1 //从1开始计数
	endIndex := page*this.pageSize - 1
	this.logger.Debug(request, startIndex, endIndex)
	keys, err := this.client.LRange(request, startIndex, endIndex).Result()

	if err != nil || len(keys) == 0 {
		if err != nil {
			this.logger.Debug(err.Error())
		}
		this.logger.Debug("no content by page!")
		return nil
	}
	go this.client.Expire(request, time.Duration(this.pageLife)*time.Minute) //更新重置失效时间
	return &PageSearchResult{request, page, this.fetch(name, keys...)}
}

//获取内容
func (this *SearchEngine) fetch(name string, keys ...string) []TargetObject {
	datas, err1 := this.client.HMGet(name, keys...).Result()
	if err1 == nil && len(datas) > 0 {

		var targets []TargetObject

		for _, v := range datas {
			if v != nil {
				t := TargetObject{}
				json.Unmarshal([]byte(fmt.Sprintf("%v", v)), &t)
				targets = append(targets, t)
			}
		}

		return targets

	} else {
		this.logger.Error("get data by index error!%s", err1.Error())
	}

	return nil
}

func (this *SearchEngine) Search(name string, input ...Field) *PageSearchResult {
	if name != "" {
		index := this.indexs[name]
		if index != nil {
			requestkey := index.Search(input...)
			if requestkey != "" {
				//取出第一页的数据返回
				return this.fetchByPage(index.name, requestkey, 1)
			}

			return nil

		}
		this.logger.Info("not found index %s", name)
	}

	return nil
}

type searchIndex struct {
	name   string
	engine *SearchEngine
}

//搜索信息
func (this *searchIndex) Search(input ...Field) string {
	//生成索引搜索请求号
	requestkey := getSearchRequestUuid(this.name)

	//搜索索引
	var searchkeys []string
	var tmpkeys []string
	var notTmpkeys []string
	for _, f := range input {

		//处理not的情况
		if strings.HasPrefix(f.Key, "!") { //字段使用！表示"非"的意思，就是不等于
			k := f.Key[1:len(f.Key)]
			v := f.Value
			arrays := strings.Split(v, "|")
			if len(arrays) > 1 {
				for _, subvalue := range arrays {
					notTmpkeys = append(notTmpkeys, this.buildTheKeyByItem(k, subvalue))
				}
			} else {
				notTmpkeys = append(notTmpkeys, this.buildTheKeyByItem(k, v))
			}

			continue
		}

		//处理并集
		v := f.Value
		arrays := strings.Split(v, "|") //使用 | 分割多个取值
		if len(arrays) > 1 {
			skey := requestkey + ":" + f.Key
			var subkeys []string
			for _, subkey := range arrays {
				subkeys = append(subkeys, this.buildTheKeyByItem(f.Key, subkey))
			}

			//取并集，将结果存入临时的key中，用于后续取交集用
			this.engine.client.SUnionStore(skey, subkeys...)
			tmpkeys = append(tmpkeys, skey) //放入临时组中用于使用后删除
			searchkeys = append(searchkeys, skey)
		} else {
			searchkeys = append(searchkeys, this.buildTheKey(f))
		}

	}
	defer this.deleteTempkeys(tmpkeys) //删除临时创建的key

	var values []string
	//有差集
	if notTmpkeys != nil && len(notTmpkeys) > 1 {
		interTmpkey := requestkey + "_tmp"
		//取交集
		result := this.engine.client.SInterStore(interTmpkey, searchkeys...)
		_, err := result.Result()
		if err != nil {
			this.engine.logger.Error("inter key error!%s", err.Error())
			return ""
		}
		//取差集合，存储
		diffkeys := make([]string, len(notTmpkeys)+1)
		diffkeys[0] = interTmpkey
		copy(diffkeys[1:len(diffkeys)], notTmpkeys)

		defer this.deleteTempkeys([]string{interTmpkey}) //删除临时key
		data := this.engine.client.SDiff(diffkeys...)
		values, _ = data.Result()

	} else { //如果无差集合，则直接返回交集
		result := this.engine.client.SInter(searchkeys...)
		values, _ = result.Result()
	}

	//存储数据
	var datas []interface{}

	for _, v := range values {
		datas = append(datas, v)
	}
	this.engine.client.RPush(requestkey, datas...)

	this.engine.client.Expire(requestkey, time.Duration(this.engine.pageLife)*time.Minute) //指定x分钟后失效

	return requestkey
}

//删除某个字段等于某值得列表，也就是删除一个词条
func (this *searchIndex) RemoveKeyword(field, word string) {
	if field != "" && word != "" {
		thekey := this.buildTheKeyByItem(field, word)
		this.engine.client.Del(thekey)
	}

}

//删除临时创建的key
func (this *searchIndex) deleteTempkeys(keys []string) {
	if keys != nil && len(keys) > 0 {
		this.engine.client.Del(keys...)
	}
}

//刷新索引，加载信息到存储中
func (this *searchIndex) LoadObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key := obj.Id
	//如果没有指明对象id，则使用md5作为数据的唯一标识
	if key == "" {
		key = getMd5str(string(data))
	}

	//1、放入数据到目标集合中
	this.engine.client.HSet(this.name, key, string(data))

	//2、根据field存储到各个对应的索引中
	for k, v := range obj.Fields {
		for _, vkey := range v {
			this.engine.client.SAdd(this.buildTheKeyByItem(k, vkey), key)
		}

	}

}

func (this *searchIndex) RemoveFromFieldById(field, value string, ids ...string) {
	var target []interface{}
	for _, id := range ids {
		target = append(target, id)
	}
	this.engine.client.SRem(this.buildTheKeyByItem(field, value), target)
}

//删除数据：删除数据及索引存的值
func (this *searchIndex) RemoveObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key := obj.Id
	if key == "" {
		key = getMd5str(string(data))
	}
	//删除数据
	this.engine.client.HDel(this.name, key)
	//删除索引里的记录
	for k, v := range obj.Fields {
		for _, vkey := range v {
			this.engine.client.SRem(this.buildTheKeyByItem(k, vkey), key)
		}
	}

}

func (this *searchIndex) buildTheKey(f Field) string {
	return this.buildTheKeyByItem(f.Key, f.Value)
}

func (this *searchIndex) buildTheKeyByItem(key, value string) string {
	return fmt.Sprintf("%s_%s_%s", this.name, key, value)
}

func getMd5str(value string) string {
	data := []byte(value)
	has := md5.Sum(data)
	md5str1 := fmt.Sprintf("%x", has) //将[]byte转成16进制

	return md5str1

}

func getSearchRequestUuid(prefix string) string {
	return fmt.Sprintf("%s:%d", prefix, time.Now().UnixNano())
}
<|endoftext|>


<commit_before>package pezinventory

import (
	"fmt"
	"os"

	"gopkg.in/mgo.v2"

	"github.com/cloudfoundry-community/go-cfenv"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/pivotal-pez/pezinventory/service/integrations"

	"github.com/unrolled/render"
)

var formatter *render.Render = nil

//Formatter returns the address for a global response formatter
//realized in the `github.com/unrolled/render` package.
func Formatter() *render.Render {
	if formatter == nil {
		formatter = render.New(render.Options{
			IndentJSON: true,
		})
	}
	return formatter
}

//NewServer configures and returns a Server.
func NewServer(appEnv *cfenv.App) *negroni.Negroni {

	inventoryServiceName := os.Getenv("INVENTORY_DB_NAME")
	inventoryServiceURIName := os.Getenv("INVENTORY_DB_URI")
	inventoryServiceURI := getServiceBinding(inventoryServiceName, inventoryServiceURIName, appEnv)
	inventoryCollection := SetupDB(integrations.NewCollectionDialer, inventoryServiceURI, InventoryCollectionName)
	leaseServiceName := os.Getenv("LEASE_DB_NAME")
	leaseServiceURIName := os.Getenv("LEASE_DB_URI")
	leaseServiceURI := getServiceBinding(leaseServiceName, leaseServiceURIName, appEnv)
	leaseCollection := SetupDB(integrations.NewCollectionDialer, leaseServiceURI, LeaseCollectionName)

	n := negroni.Classic()
	mx := mux.NewRouter()

	mx.HandleFunc("/v1/inventory", ListInventoryItemsHandler(inventoryCollection)).Methods("GET")
	mx.HandleFunc("/v1/inventory", InsertInventoryItemHandler(inventoryCollection)).Methods("POST")
	mx.HandleFunc("/v1/lease/{id}", FindLeaseByIDHandler(leaseCollection)).Methods("GET")
	mx.HandleFunc("/v1/lease", InsertLeaseRecordHandler(leaseCollection)).Methods("POST")

	n.UseHandler(mx)

	return n
}

func getServiceBinding(serviceName string, serviceURIName string, appEnv *cfenv.App) (serviceURI string) {

	if service, err := appEnv.Services.WithName(serviceName); err == nil {
		if serviceURI = service.Credentials[serviceURIName].(string); serviceURI == "" {
			panic(fmt.Sprintf("we pulled an empty connection string %s from %v - %v", serviceURI, service, service.Credentials))
		}

	} else {
		panic(fmt.Sprint("Experienced an error trying to grab service binding information:", err.Error()))
	}
	return
}

func SetupDB(dialer integrations.CollectionDialer, URI string, collectionName string) (collection integrations.Collection) {
	var (
		err      error
		dialInfo *mgo.DialInfo
	)

	if dialInfo, err = mgo.ParseURL(URI); err != nil || dialInfo.Database == "" {
		panic(fmt.Sprintf("can not parse given URI %s due to error: %s", URI, err.Error()))
	}

	if collection, err = dialer(URI, dialInfo.Database, collectionName); err != nil {
		panic(fmt.Sprintf("can not dial connection due to error: %s URI:%s col:%s db:%s", err.Error(), URI, collectionName, dialInfo.Database))
	}
	return
}
<commit_msg>[#101295544] Attempting multiple collections in single mongodb service.<commit_after>package pezinventory

import (
	"fmt"
	"os"

	"gopkg.in/mgo.v2"

	"github.com/cloudfoundry-community/go-cfenv"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/pivotal-pez/pezinventory/service/integrations"

	"github.com/unrolled/render"
)

var formatter *render.Render = nil

//Formatter returns the address for a global response formatter
//realized in the `github.com/unrolled/render` package.
func Formatter() *render.Render {
	if formatter == nil {
		formatter = render.New(render.Options{
			IndentJSON: true,
		})
	}
	return formatter
}

//NewServer configures and returns a Server.
func NewServer(appEnv *cfenv.App) *negroni.Negroni {

	inventoryServiceName := os.Getenv("INVENTORY_DB_NAME")
	inventoryServiceURIName := os.Getenv("INVENTORY_DB_URI")
	inventoryServiceURI := getServiceBinding(inventoryServiceName, inventoryServiceURIName, appEnv)
	inventoryCollection := SetupDB(integrations.NewCollectionDialer, inventoryServiceURI, InventoryCollectionName)
	// leaseServiceName := os.Getenv("LEASE_DB_NAME")
	// leaseServiceURIName := os.Getenv("LEASE_DB_URI")
	// leaseServiceURI := getServiceBinding(leaseServiceName, leaseServiceURIName, appEnv)
	leaseCollection := SetupDB(integrations.NewCollectionDialer, inventoryServiceURI, LeaseCollectionName)

	n := negroni.Classic()
	mx := mux.NewRouter()

	mx.HandleFunc("/v1/inventory", ListInventoryItemsHandler(inventoryCollection)).Methods("GET")
	mx.HandleFunc("/v1/inventory", InsertInventoryItemHandler(inventoryCollection)).Methods("POST")
	mx.HandleFunc("/v1/lease/{id}", FindLeaseByIDHandler(leaseCollection)).Methods("GET")
	mx.HandleFunc("/v1/lease", InsertLeaseRecordHandler(leaseCollection)).Methods("POST")

	n.UseHandler(mx)

	return n
}

func getServiceBinding(serviceName string, serviceURIName string, appEnv *cfenv.App) (serviceURI string) {

	if service, err := appEnv.Services.WithName(serviceName); err == nil {
		if serviceURI = service.Credentials[serviceURIName].(string); serviceURI == "" {
			panic(fmt.Sprintf("we pulled an empty connection string %s from %v - %v", serviceURI, service, service.Credentials))
		}

	} else {
		panic(fmt.Sprint("Experienced an error trying to grab service binding information:", err.Error()))
	}
	return
}

func SetupDB(dialer integrations.CollectionDialer, URI string, collectionName string) (collection integrations.Collection) {
	var (
		err      error
		dialInfo *mgo.DialInfo
	)

	if dialInfo, err = mgo.ParseURL(URI); err != nil || dialInfo.Database == "" {
		panic(fmt.Sprintf("can not parse given URI %s due to error: %s", URI, err.Error()))
	}

	if collection, err = dialer(URI, dialInfo.Database, collectionName); err != nil {
		panic(fmt.Sprintf("can not dial connection due to error: %s URI:%s col:%s db:%s", err.Error(), URI, collectionName, dialInfo.Database))
	}
	return
}
<|endoftext|>


<commit_before>package config

import (
	"io/ioutil"
	"os"
	"strconv"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/omakoto/zenlog-go/zenlog/envs"
	"github.com/omakoto/zenlog-go/zenlog/util"
	"runtime"
)

// Config represents configuration parameters, read from ~/.zenlog.toml and overridden with the environmental variables.
type Config struct {
	LogDir              string `toml:"ZENLOG_DIR"`
	TempDir             string `toml:"ZENLOG_TEMP"`
	StartCommand        string `toml:"ZENLOG_START_COMMAND"`
	PrefixCommands      string `toml:"ZENLOG_PREFIX_COMMANDS"`
	AlwaysNoLogCommands string `toml:"ZENLOG_ALWAYS_NO_LOG_COMMANDS"`
	AutoFlush           bool   `toml:"ZENLOG_AUTO_FLUSH"`

	UseExperimentalCommandParser bool   `toml:"ZENLOG_USE_EXPERIMENTAL_COMMAND_PARSER"`
	CommandSplitter              string `toml:"ZENLOG_COMMAND_SPLITTER"`
	CommentSplitter              string `toml:"ZENLOG_COMMENT_SPLITTER"`

	Maxproc int `toml:"ZENLOG_MAXPROC"`

	UseSplice bool `toml:"ZENLOG_USE_SPLICE"`

	ZenlogPid int

	OuterTty  string
	LoggerIn  string
	LoggerOut string
}

// If a string is empty, get from a given environmental variabne.
func overwriteWithEnviron(to *string, envKey string, def string) {
	if val := os.Getenv(envKey); val != "" {
		*to = val
	} else {
		*to = os.ExpandEnv(*to)
	}
	if *to == "" {
		*to = def
	}
}

func overwriteBoolWithEnviron(to *bool, envKey string) {
	if val := os.Getenv(envKey); val != "" {
		*to = (val == "1")
	}
}

// Make sure a string ends with a slash.
func ensureSlash(v *string) {
	if strings.HasSuffix(*v, "/") {
		return
	}
	*v = *v + "/"
}

// InitConfigForLogger returns a Config for a new session, loading from ~/.zenlog.toml and the environmental variables.
func InitConfigForLogger() *Config {
	file := os.Getenv(envs.ZenlogConf)
	if file == "" {
		file = os.ExpandEnv("$HOME/.zenlog.toml")
	}
	var c Config

	c.UseExperimentalCommandParser = true // Default to true.

	data, err := ioutil.ReadFile(file)
	if err == nil {
		if _, err := toml.Decode(string(data), &c); err != nil {
			util.Fatalf("Unable to load %s: %s", file, err)
		}
	} else if os.IsNotExist(err) {
		util.Warn(err, "%s doesn't exist; using the default instead", file)
	}

	overwriteWithEnviron(&c.StartCommand, "ZENLOG_START_COMMAND", "")
	overwriteWithEnviron(&c.LogDir, envs.ZenlogDir, os.ExpandEnv("$HOME/zenlog/"))
	overwriteWithEnviron(&c.PrefixCommands, "ZENLOG_PREFIX_COMMANDS", `(?:command|builtin|time|sudo|[a-zA-Z0-9_]+\=.*)`)
	overwriteWithEnviron(&c.AlwaysNoLogCommands, "ZENLOG_ALWAYS_NO_LOG_COMMANDS", `(?:vi|vim|man|nano|pico|emacs|zenlog.*)`)

	overwriteWithEnviron(&c.CommandSplitter, "ZENLOG_COMMAND_SPLITTER", "")
	overwriteWithEnviron(&c.CommentSplitter, "ZENLOG_COMMENT_SPLITTER", "")

	overwriteWithEnviron(&c.TempDir, envs.ZenlogTemp, "")

	overwriteBoolWithEnviron(&c.AutoFlush, envs.ZenlogAutoFlush)
	overwriteBoolWithEnviron(&c.UseExperimentalCommandParser, envs.ZenlogUseExperimentalCommandParser)

	if c.Maxproc < 1 {
		c.Maxproc = 1
	}

	if c.StartCommand == "" {
		shell := os.Getenv("SHELL")
		if shell != "" {
			c.StartCommand = "exec " + shell + " -l"
		} else {
			c.StartCommand = "exec /bin/sh -l"
		}
	}

	if c.TempDir == "" || !util.DirExists(c.TempDir) {
		c.TempDir = util.FirstNonEmpty(os.Getenv("TEMP"), os.Getenv("TMP"), "/tmp/")
	}

	ensureSlash(&c.LogDir)
	ensureSlash(&c.TempDir)

	// For E2E testing, override the PID with _ZENLOG_LOGGER_PID, if set.
	c.ZenlogPid = util.GetIntEnv("_ZENLOG_LOGGER_PID", os.Getpid())

	runtime.GOMAXPROCS(c.Maxproc)

	util.Dump("Logger config=", c)

	return &c
}

// InitConfigForCommands returns a Config for subcommands, loading from ~/.zenlog.toml and the environmental variables.
// Some of the parameters (such as ZenlogPid) will be inherited from the current zenlog session.
func InitConfigForCommands() *Config {
	var c Config

	// Take over some of the parameters from the logger.

	pid, err := strconv.Atoi(os.Getenv(envs.ZenlogPid))
	util.Check(err, "ZENLOG_PID not integer")
	c.ZenlogPid = pid

	c.LogDir = os.Getenv(envs.ZenlogDir)
	c.OuterTty = os.Getenv(envs.ZenlogOuterTty)
	c.LoggerIn = os.Getenv(envs.ZenlogLoggerIn)
	c.LoggerOut = os.Getenv(envs.ZenlogLoggerOut)

	if c.ZenlogPid == 0 {
		util.Fatalf(envs.ZenlogPid + " not set.")
	}
	if c.LogDir == "" {
		util.Fatalf(envs.ZenlogDir + " not set.")
	}
	if c.OuterTty == "" {
		util.Fatalf(envs.ZenlogOuterTty + " not set.")
	}
	if c.LoggerIn == "" {
		util.Fatalf(envs.ZenlogLoggerIn + " not set.")
	}
	if c.LoggerOut == "" {
		util.Fatalf(envs.ZenlogLoggerOut + " not set.")
	}

	// We still need to load certain parameters from TOML.
	lc := InitConfigForLogger()
	c.AlwaysNoLogCommands = lc.AlwaysNoLogCommands
	c.PrefixCommands = lc.PrefixCommands
	c.CommandSplitter = lc.CommandSplitter
	c.CommentSplitter = lc.CommentSplitter
	c.UseExperimentalCommandParser = lc.UseExperimentalCommandParser
	c.TempDir = lc.TempDir
	c.Maxproc = lc.Maxproc

	util.Dump("Command config=", c)
	return &c
}
<commit_msg>UseSplice.<commit_after>package config

import (
	"io/ioutil"
	"os"
	"strconv"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/omakoto/zenlog-go/zenlog/envs"
	"github.com/omakoto/zenlog-go/zenlog/util"
	"runtime"
)

// Config represents configuration parameters, read from ~/.zenlog.toml and overridden with the environmental variables.
type Config struct {
	LogDir              string `toml:"ZENLOG_DIR"`
	TempDir             string `toml:"ZENLOG_TEMP"`
	StartCommand        string `toml:"ZENLOG_START_COMMAND"`
	PrefixCommands      string `toml:"ZENLOG_PREFIX_COMMANDS"`
	AlwaysNoLogCommands string `toml:"ZENLOG_ALWAYS_NO_LOG_COMMANDS"`
	AutoFlush           bool   `toml:"ZENLOG_AUTO_FLUSH"`

	UseExperimentalCommandParser bool   `toml:"ZENLOG_USE_EXPERIMENTAL_COMMAND_PARSER"`
	CommandSplitter              string `toml:"ZENLOG_COMMAND_SPLITTER"`
	CommentSplitter              string `toml:"ZENLOG_COMMENT_SPLITTER"`

	Maxproc int `toml:"ZENLOG_MAXPROC"`

	UseSplice bool `toml:"ZENLOG_USE_SPLICE"`

	ZenlogPid int

	OuterTty  string
	LoggerIn  string
	LoggerOut string
}

// If a string is empty, get from a given environmental variabne.
func overwriteWithEnviron(to *string, envKey string, def string) {
	if val := os.Getenv(envKey); val != "" {
		*to = val
	} else {
		*to = os.ExpandEnv(*to)
	}
	if *to == "" {
		*to = def
	}
}

func overwriteBoolWithEnviron(to *bool, envKey string) {
	if val := os.Getenv(envKey); val != "" {
		*to = (val == "1")
	}
}

// Make sure a string ends with a slash.
func ensureSlash(v *string) {
	if strings.HasSuffix(*v, "/") {
		return
	}
	*v = *v + "/"
}

// InitConfigForLogger returns a Config for a new session, loading from ~/.zenlog.toml and the environmental variables.
func InitConfigForLogger() *Config {
	file := os.Getenv(envs.ZenlogConf)
	if file == "" {
		file = os.ExpandEnv("$HOME/.zenlog.toml")
	}
	var c Config

	c.UseExperimentalCommandParser = true // Default to true.

	data, err := ioutil.ReadFile(file)
	if err == nil {
		if _, err := toml.Decode(string(data), &c); err != nil {
			util.Fatalf("Unable to load %s: %s", file, err)
		}
	} else if os.IsNotExist(err) {
		util.Warn(err, "%s doesn't exist; using the default instead", file)
	}

	overwriteWithEnviron(&c.StartCommand, "ZENLOG_START_COMMAND", "")
	overwriteWithEnviron(&c.LogDir, envs.ZenlogDir, os.ExpandEnv("$HOME/zenlog/"))
	overwriteWithEnviron(&c.PrefixCommands, "ZENLOG_PREFIX_COMMANDS", `(?:command|builtin|time|sudo|[a-zA-Z0-9_]+\=.*)`)
	overwriteWithEnviron(&c.AlwaysNoLogCommands, "ZENLOG_ALWAYS_NO_LOG_COMMANDS", `(?:vi|vim|man|nano|pico|emacs|zenlog.*)`)

	overwriteWithEnviron(&c.CommandSplitter, "ZENLOG_COMMAND_SPLITTER", "")
	overwriteWithEnviron(&c.CommentSplitter, "ZENLOG_COMMENT_SPLITTER", "")

	overwriteWithEnviron(&c.TempDir, envs.ZenlogTemp, "")

	overwriteBoolWithEnviron(&c.AutoFlush, envs.ZenlogAutoFlush)
	overwriteBoolWithEnviron(&c.UseExperimentalCommandParser, envs.ZenlogUseExperimentalCommandParser)
	overwriteBoolWithEnviron(&c.UseSplice, envs.ZenlogUseSplice)

	if c.Maxproc < 1 {
		c.Maxproc = 1
	}

	if c.StartCommand == "" {
		shell := os.Getenv("SHELL")
		if shell != "" {
			c.StartCommand = "exec " + shell + " -l"
		} else {
			c.StartCommand = "exec /bin/sh -l"
		}
	}

	if c.TempDir == "" || !util.DirExists(c.TempDir) {
		c.TempDir = util.FirstNonEmpty(os.Getenv("TEMP"), os.Getenv("TMP"), "/tmp/")
	}

	ensureSlash(&c.LogDir)
	ensureSlash(&c.TempDir)

	// For E2E testing, override the PID with _ZENLOG_LOGGER_PID, if set.
	c.ZenlogPid = util.GetIntEnv("_ZENLOG_LOGGER_PID", os.Getpid())

	runtime.GOMAXPROCS(c.Maxproc)

	util.Dump("Logger config=", c)

	return &c
}

// InitConfigForCommands returns a Config for subcommands, loading from ~/.zenlog.toml and the environmental variables.
// Some of the parameters (such as ZenlogPid) will be inherited from the current zenlog session.
func InitConfigForCommands() *Config {
	var c Config

	// Take over some of the parameters from the logger.

	pid, err := strconv.Atoi(os.Getenv(envs.ZenlogPid))
	util.Check(err, "ZENLOG_PID not integer")
	c.ZenlogPid = pid

	c.LogDir = os.Getenv(envs.ZenlogDir)
	c.OuterTty = os.Getenv(envs.ZenlogOuterTty)
	c.LoggerIn = os.Getenv(envs.ZenlogLoggerIn)
	c.LoggerOut = os.Getenv(envs.ZenlogLoggerOut)

	if c.ZenlogPid == 0 {
		util.Fatalf(envs.ZenlogPid + " not set.")
	}
	if c.LogDir == "" {
		util.Fatalf(envs.ZenlogDir + " not set.")
	}
	if c.OuterTty == "" {
		util.Fatalf(envs.ZenlogOuterTty + " not set.")
	}
	if c.LoggerIn == "" {
		util.Fatalf(envs.ZenlogLoggerIn + " not set.")
	}
	if c.LoggerOut == "" {
		util.Fatalf(envs.ZenlogLoggerOut + " not set.")
	}

	// We still need to load certain parameters from TOML.
	lc := InitConfigForLogger()
	c.AlwaysNoLogCommands = lc.AlwaysNoLogCommands
	c.PrefixCommands = lc.PrefixCommands
	c.CommandSplitter = lc.CommandSplitter
	c.CommentSplitter = lc.CommentSplitter
	c.UseExperimentalCommandParser = lc.UseExperimentalCommandParser
	c.TempDir = lc.TempDir
	c.Maxproc = lc.Maxproc

	util.Dump("Command config=", c)
	return &c
}
<|endoftext|>