<commit_before>package rest

import (
	"encoding/json"
	"log"
	"os"
	"time"
)

// AccessLogJsonMiddleware produces the access log with records written as JSON.
// It depends on the timer, recorder and auth middlewares.
type AccessLogJsonMiddleware struct {
	Logger *log.Logger
}

// MiddlewareFunc makes AccessLogJsonMiddleware implement the Middleware interface.
func (mw *AccessLogJsonMiddleware) MiddlewareFunc(h HandlerFunc) HandlerFunc {

	// set the default Logger
	if mw.Logger == nil {
		mw.Logger = log.New(os.Stderr, "", 0)
	}

	return func(w ResponseWriter, r *Request) {

		// call the handler
		h(w, r)

		mw.Logger.Printf("%s", makeAccessLogJsonRecord(r).asJson())
	}
}

// When EnableLogAsJson is true, this object is dumped as JSON in the Logger.
// (Public for documentation only, no public method uses it).
type AccessLogJsonRecord struct {
	Timestamp    *time.Time
	StatusCode   int
	ResponseTime *time.Duration
	HttpMethod   string
	RequestURI   string
	RemoteUser   string
	UserAgent    string
}

func makeAccessLogJsonRecord(r *Request) *AccessLogJsonRecord {

	var timestamp *time.Time
	if r.Env["START_TIME"] != nil {
		timestamp = r.Env["START_TIME"].(*time.Time)
	}

	var statusCode int
	if r.Env["STATUS_CODE"] != nil {
		statusCode = r.Env["STATUS_CODE"].(int)
	}

	var responseTime *time.Duration
	if r.Env["ELAPSED_TIME"] != nil {
		responseTime = r.Env["ELAPSED_TIME"].(*time.Duration)
	}

	var remoteUser string
	if r.Env["REMOTE_USER"] != nil {
		remoteUser = r.Env["REMOTE_USER"].(string)
	}

	return &AccessLogJsonRecord{
		Timestamp:    timestamp,
		StatusCode:   statusCode,
		ResponseTime: responseTime,
		HttpMethod:   r.Method,
		RequestURI:   r.URL.RequestURI(),
		RemoteUser:   remoteUser,
		UserAgent:    r.UserAgent(),
	}
}

func (r *AccessLogJsonRecord) asJson() []byte {
	b, err := json.Marshal(r)
	if err != nil {
		panic(err)
	}
	return b
}
<commit_msg>AccessLogJsonMiddlesware docstrings<commit_after>package rest

import (
	"encoding/json"
	"log"
	"os"
	"time"
)

// AccessLogJsonMiddleware produces the access log with records written as JSON.
// Logger defaults to log.New(os.Stderr, "", 0). This middleware depends on TimerMiddleware
// and RecorderMiddleware. It also uses request.Env["REMOTE_USER"].(string) set by the auth
// middlewares.
type AccessLogJsonMiddleware struct {
	Logger *log.Logger
}

// MiddlewareFunc makes AccessLogJsonMiddleware implement the Middleware interface.
func (mw *AccessLogJsonMiddleware) MiddlewareFunc(h HandlerFunc) HandlerFunc {

	// set the default Logger
	if mw.Logger == nil {
		mw.Logger = log.New(os.Stderr, "", 0)
	}

	return func(w ResponseWriter, r *Request) {

		// call the handler
		h(w, r)

		mw.Logger.Printf("%s", makeAccessLogJsonRecord(r).asJson())
	}
}

// AccessLogJsonRecord is the data structure used by AccessLogJsonMiddleware to create the JSON
// records. (Public for documentation only, no public method uses it)
type AccessLogJsonRecord struct {
	Timestamp    *time.Time
	StatusCode   int
	ResponseTime *time.Duration
	HttpMethod   string
	RequestURI   string
	RemoteUser   string
	UserAgent    string
}

func makeAccessLogJsonRecord(r *Request) *AccessLogJsonRecord {

	var timestamp *time.Time
	if r.Env["START_TIME"] != nil {
		timestamp = r.Env["START_TIME"].(*time.Time)
	}

	var statusCode int
	if r.Env["STATUS_CODE"] != nil {
		statusCode = r.Env["STATUS_CODE"].(int)
	}

	var responseTime *time.Duration
	if r.Env["ELAPSED_TIME"] != nil {
		responseTime = r.Env["ELAPSED_TIME"].(*time.Duration)
	}

	var remoteUser string
	if r.Env["REMOTE_USER"] != nil {
		remoteUser = r.Env["REMOTE_USER"].(string)
	}

	return &AccessLogJsonRecord{
		Timestamp:    timestamp,
		StatusCode:   statusCode,
		ResponseTime: responseTime,
		HttpMethod:   r.Method,
		RequestURI:   r.URL.RequestURI(),
		RemoteUser:   remoteUser,
		UserAgent:    r.UserAgent(),
	}
}

func (r *AccessLogJsonRecord) asJson() []byte {
	b, err := json.Marshal(r)
	if err != nil {
		panic(err)
	}
	return b
}
<|endoftext|>


<commit_before>package openrtb2

import "encoding/json"

// 3.2.18 Object: Device
//
// This object provides information pertaining to the device through which the user is interacting.
// Device information includes its hardware, platform, location, and carrier data.
// The device can refer to a mobile handset, a desktop computer, set top box, or other digital device.
//
// BEST PRACTICE: There are currently no prominent open source lists for device makes, models, operating systems, or carriers.
// Exchanges typically use commercial products or other proprietary lists for these attributes.
// Until suitable open standards are available, exchanges are highly encouraged to publish lists of their device make, model, operating system, and carrier values to bidders.
//
// BEST PRACTICE: Proper device IP detection in mobile is not straightforward.
// Typically it involves starting at the left of the x-forwarded-for header, skipping private carrier networks (e.g., 10.x.x.x or 192.x.x.x), and possibly scanning for known carrier IP ranges.
// Exchanges are urged to research and implement this feature carefully when presenting device IP values to bidders.
type Device struct {

	// Attribute:
	//   geo
	// Type:
	//   object; recommended
	// Description:
	//   Location of the device assumed to be the user’s current
	//   location defined by a Geo object (Section 3.2.19).
	Geo *Geo `json:"geo,omitempty"`

	// Attribute:
	//   dnt
	// Type:
	//   integer; recommended
	// Description:
	//   Standard “Do Not Track” flag as set in the header by the
	//   browser, where 0 = tracking is unrestricted, 1 = do not track.
	DNT *int8 `json:"dnt,omitempty"`

	// Attribute:
	//   lmt
	// Type:
	//   integer; recommended
	// Description:
	//   “Limit Ad Tracking” signal commercially endorsed (e.g., iOS,
	//   Android), where 0 = tracking is unrestricted, 1 = tracking must
	//   be limited per commercial guidelines.
	Lmt *int8 `json:"lmt,omitempty"`

	// Attribute:
	//   ua
	// Type:
	//   string
	// Description:
	//   Browser user agent string. This field represents a raw user
	//   agent string from the browser. For backwards compatibility,
	//   exchanges are recommended to always populate ‘ua’ with the
	//   User-Agent string, when available from the end user’s device,
	//   even if an alternative representation, such as the User-Agent
	//   Client-Hints, is available and is used to populate ‘sua’. No
	//   inferred or approximated user agents are expected in this field.
	//   If a client supports User-Agent Client Hints, and ‘sua’ field is
	//   present, bidders are recommended to rely on ‘sua’ for
	//   detecting device type, browser type and version and other
	//   purposes that rely on the user agent information, and ignore
	//   ‘ua’ field. This is because the ‘ua’ may contain a frozen or
	//   reduced user agent string.
	UA string `json:"ua,omitempty"`

	// Attribute:
	//   sua
	// Type:
	//   object
	// Description:
	//   Structured user agent information defined by a UserAgent
	//   object (see Section 3.2.29). If both ‘ua’ and ‘sua’ are present in
	//   the bid request, ‘sua’ should be considered the more accurate
	//   representation of the device attributes. This is because the ‘ua’
	//   may contain a frozen or reduced user agent string.
	SUA *UserAgent `json:"sua,omitempty"`

	// Attribute:
	//   ip
	// Type:
	//   string; recommended
	// Description:
	//   IPv4 address closest to device.
	IP string `json:"ip,omitempty"`

	// Attribute:
	//   ipv6
	// Type:
	//   string
	// Description:
	//   IP address closest to device as IPv6.
	IPv6 string `json:"ipv6,omitempty"`

	// Attribute:
	//   devicetype
	// Type:
	//   integer
	// Description:
	//   The general type of device. Refer to List: Device Types in
	//   AdCOM 1.0.
	DeviceType int8 `json:"devicetype,omitempty"`

	// Attribute:
	//   make
	// Type:
	//   string
	// Description:
	//   Device make (e.g., “Apple”).
	Make string `json:"make,omitempty"`

	// Attribute:
	//   model
	// Type:
	//   string
	// Description:
	//   Device model (e.g., “iPhone”).
	Model string `json:"model,omitempty"`

	// Attribute:
	//   os
	// Type:
	//   string
	// Description:
	//   Device operating system (e.g., “iOS”).
	OS string `json:"os,omitempty"`

	// Attribute:
	//   osv
	// Type:
	//   string
	// Description:
	//   Device operating system version (e.g., “3.1.2”).
	OSV string `json:"osv,omitempty"`

	// Attribute:
	//   hwv
	// Type:
	//   string
	// Description:
	//   Hardware version of the device (e.g., “5S” for iPhone 5S).
	HWV string `json:"hwv,omitempty"`

	// Attribute:
	//   h
	// Type:
	//   integer
	// Description:
	//   Physical height of the screen in pixels.
	H int64 `json:"h,omitempty"`

	// Attribute:
	//   w
	// Type:
	//   integer
	// Description:
	//   Physical width of the screen in pixels.
	W int64 `json:"w,omitempty"`

	// Attribute:
	//   ppi
	// Type:
	//   integer
	// Description:
	//   Screen size as pixels per linear inch.
	PPI int64 `json:"ppi,omitempty"`

	// Attribute:
	//   pxratio
	// Type:
	//   float
	// Description:
	//   The ratio of physical pixels to device independent pixels.
	PxRatio float64 `json:"pxratio,omitempty"`

	// Attribute:
	//   js
	// Type:
	//   integer
	// Description:
	//   Support for JavaScript, where 0 = no, 1 = yes.
	JS int8 `json:"js,omitempty"`

	// Attribute:
	//   geofetch
	// Type:
	//   integer
	// Description:
	//   Indicates if the geolocation API will be available to JavaScript
	//   code running in the banner, where 0 = no, 1 = yes.
	GeoFetch int8 `json:"geofetch,omitempty"`

	// Attribute:
	//   flashver
	// Type:
	//   string
	// Description:
	//   Version of Flash supported by the browser.
	FlashVer string `json:"flashver,omitempty"`

	// Attribute:
	//   language
	// Type:
	//   string
	// Description:
	//   Browser language using ISO-639-1-alpha-2. Only one of
	//   language or langb should be present.
	Language string `json:"language,omitempty"`

	// Attribute:
	//   langb
	// Type:
	//   string
	// Description:
	//   Content language using IETF BCP 47. Only one of language or
	//   langb should be present.
	LangB string `json:"langb,omitempty"`

	// Attribute:
	//   carrier
	// Type:
	//   string
	// Description:
	//   Carrier or ISP (e.g., “VERIZON”) using exchange curated string
	//   names which should be published to bidders a priori.
	Carrier string `json:"carrier,omitempty"`

	// Attribute:
	//   mccmnc
	// Type:
	//   string
	// Description:
	//   Mobile carrier as the concatenated MCC-MNC code
	//   (e.g., "310-005" identifies Verizon Wireless CDMA in the
	//   USA).
	//   Refer to https://en.wikipedia.org/wiki/Mobile_country_code
	//   for further examples. Note that the dash between the MCC
	//   and MNC parts is required to remove parsing ambiguity. The
	//   MCC-MNC values represent the SIM installed on the device
	//   and do not change when a device is roaming. Roaming may
	//   be inferred by a combination of the MCC-MNC, geo, IP and
	//   other data signals.
	MCCMNC string `json:"mccmnc,omitempty"`

	// Attribute:
	//   connectiontype
	// Type:
	//   integer
	// Description:
	//   Network connection type. Refer to List: Connection Types in
	//   AdCOM 1.0.
	ConnectionType int8 `json:"connectiontype,omitempty"`

	// Attribute:
	//   ifa
	// Type:
	//   string
	// Description:
	//   ID sanctioned for advertiser use in the clear (i.e., not hashed).
	IFA string `json:"ifa,omitempty"`

	// Attribute:
	//   didsha1
	// Type:
	//   string; DEPRECATED
	// Description:
	//   Hardware device ID (e.g., IMEI); hashed via SHA1.
	DIDSHA1 string `json:"didsha1,omitempty"`

	// Attribute:
	//   didmd5
	// Type:
	//   string; DEPRECATED
	// Description:
	//  Hardware device ID (e.g., IMEI); hashed via MD5.
	DIDMD5 string `json:"didmd5,omitempty"`

	// Attribute:
	//   dpidsha1
	// Type:
	//   string; DEPRECATED
	// Description:
	//   Platform device ID (e.g., Android ID); hashed via SHA1.
	DPIDSHA1 string `json:"dpidsha1,omitempty"`

	// Attribute:
	//   dpidmd5
	// Type:
	//   string; DEPRECATED
	// Description:
	//   Platform device ID (e.g., Android ID); hashed via MD5.
	DPIDMD5 string `json:"dpidmd5,omitempty"`

	// Attribute:
	//   macsha1
	// Type:
	//   string; DEPRECATED
	// Description:
	//   MAC address of the device; hashed via SHA1.
	MACSHA1 string `json:"macsha1,omitempty"`

	// Attribute:
	//   macmd5
	// Type:
	//   string; DEPRECATED
	// Description:
	//   MAC address of the device; hashed via MD5.
	MACMD5 string `json:"macmd5,omitempty"`

	// Attribute:
	//   ext
	// Type:
	//   object
	// Description:
	//   Placeholder for exchange-specific extensions to OpenRTB.
	Ext json.RawMessage `json:"ext,omitempty"`
}
<commit_msg>DeviceType Enum<commit_after>package openrtb2

import (
	"encoding/json"

	"github.com/mxmCherry/openrtb/v16/adcom1"
)

// 3.2.18 Object: Device
//
// This object provides information pertaining to the device through which the user is interacting.
// Device information includes its hardware, platform, location, and carrier data.
// The device can refer to a mobile handset, a desktop computer, set top box, or other digital device.
//
// BEST PRACTICE: There are currently no prominent open source lists for device makes, models, operating systems, or carriers.
// Exchanges typically use commercial products or other proprietary lists for these attributes.
// Until suitable open standards are available, exchanges are highly encouraged to publish lists of their device make, model, operating system, and carrier values to bidders.
//
// BEST PRACTICE: Proper device IP detection in mobile is not straightforward.
// Typically it involves starting at the left of the x-forwarded-for header, skipping private carrier networks (e.g., 10.x.x.x or 192.x.x.x), and possibly scanning for known carrier IP ranges.
// Exchanges are urged to research and implement this feature carefully when presenting device IP values to bidders.
type Device struct {

	// Attribute:
	//   geo
	// Type:
	//   object; recommended
	// Description:
	//   Location of the device assumed to be the user’s current
	//   location defined by a Geo object (Section 3.2.19).
	Geo *Geo `json:"geo,omitempty"`

	// Attribute:
	//   dnt
	// Type:
	//   integer; recommended
	// Description:
	//   Standard “Do Not Track” flag as set in the header by the
	//   browser, where 0 = tracking is unrestricted, 1 = do not track.
	DNT *int8 `json:"dnt,omitempty"`

	// Attribute:
	//   lmt
	// Type:
	//   integer; recommended
	// Description:
	//   “Limit Ad Tracking” signal commercially endorsed (e.g., iOS,
	//   Android), where 0 = tracking is unrestricted, 1 = tracking must
	//   be limited per commercial guidelines.
	Lmt *int8 `json:"lmt,omitempty"`

	// Attribute:
	//   ua
	// Type:
	//   string
	// Description:
	//   Browser user agent string. This field represents a raw user
	//   agent string from the browser. For backwards compatibility,
	//   exchanges are recommended to always populate ‘ua’ with the
	//   User-Agent string, when available from the end user’s device,
	//   even if an alternative representation, such as the User-Agent
	//   Client-Hints, is available and is used to populate ‘sua’. No
	//   inferred or approximated user agents are expected in this field.
	//   If a client supports User-Agent Client Hints, and ‘sua’ field is
	//   present, bidders are recommended to rely on ‘sua’ for
	//   detecting device type, browser type and version and other
	//   purposes that rely on the user agent information, and ignore
	//   ‘ua’ field. This is because the ‘ua’ may contain a frozen or
	//   reduced user agent string.
	UA string `json:"ua,omitempty"`

	// Attribute:
	//   sua
	// Type:
	//   object
	// Description:
	//   Structured user agent information defined by a UserAgent
	//   object (see Section 3.2.29). If both ‘ua’ and ‘sua’ are present in
	//   the bid request, ‘sua’ should be considered the more accurate
	//   representation of the device attributes. This is because the ‘ua’
	//   may contain a frozen or reduced user agent string.
	SUA *UserAgent `json:"sua,omitempty"`

	// Attribute:
	//   ip
	// Type:
	//   string; recommended
	// Description:
	//   IPv4 address closest to device.
	IP string `json:"ip,omitempty"`

	// Attribute:
	//   ipv6
	// Type:
	//   string
	// Description:
	//   IP address closest to device as IPv6.
	IPv6 string `json:"ipv6,omitempty"`

	// Attribute:
	//   devicetype
	// Type:
	//   integer
	// Description:
	//   The general type of device. Refer to List: Device Types in
	//   AdCOM 1.0.
	// Note:
	//   OpenRTB <=2.5 defined only types 1..7.
	DeviceType adcom1.DeviceType `json:"devicetype,omitempty"`

	// Attribute:
	//   make
	// Type:
	//   string
	// Description:
	//   Device make (e.g., “Apple”).
	Make string `json:"make,omitempty"`

	// Attribute:
	//   model
	// Type:
	//   string
	// Description:
	//   Device model (e.g., “iPhone”).
	Model string `json:"model,omitempty"`

	// Attribute:
	//   os
	// Type:
	//   string
	// Description:
	//   Device operating system (e.g., “iOS”).
	OS string `json:"os,omitempty"`

	// Attribute:
	//   osv
	// Type:
	//   string
	// Description:
	//   Device operating system version (e.g., “3.1.2”).
	OSV string `json:"osv,omitempty"`

	// Attribute:
	//   hwv
	// Type:
	//   string
	// Description:
	//   Hardware version of the device (e.g., “5S” for iPhone 5S).
	HWV string `json:"hwv,omitempty"`

	// Attribute:
	//   h
	// Type:
	//   integer
	// Description:
	//   Physical height of the screen in pixels.
	H int64 `json:"h,omitempty"`

	// Attribute:
	//   w
	// Type:
	//   integer
	// Description:
	//   Physical width of the screen in pixels.
	W int64 `json:"w,omitempty"`

	// Attribute:
	//   ppi
	// Type:
	//   integer
	// Description:
	//   Screen size as pixels per linear inch.
	PPI int64 `json:"ppi,omitempty"`

	// Attribute:
	//   pxratio
	// Type:
	//   float
	// Description:
	//   The ratio of physical pixels to device independent pixels.
	PxRatio float64 `json:"pxratio,omitempty"`

	// Attribute:
	//   js
	// Type:
	//   integer
	// Description:
	//   Support for JavaScript, where 0 = no, 1 = yes.
	JS int8 `json:"js,omitempty"`

	// Attribute:
	//   geofetch
	// Type:
	//   integer
	// Description:
	//   Indicates if the geolocation API will be available to JavaScript
	//   code running in the banner, where 0 = no, 1 = yes.
	GeoFetch int8 `json:"geofetch,omitempty"`

	// Attribute:
	//   flashver
	// Type:
	//   string
	// Description:
	//   Version of Flash supported by the browser.
	FlashVer string `json:"flashver,omitempty"`

	// Attribute:
	//   language
	// Type:
	//   string
	// Description:
	//   Browser language using ISO-639-1-alpha-2. Only one of
	//   language or langb should be present.
	Language string `json:"language,omitempty"`

	// Attribute:
	//   langb
	// Type:
	//   string
	// Description:
	//   Content language using IETF BCP 47. Only one of language or
	//   langb should be present.
	LangB string `json:"langb,omitempty"`

	// Attribute:
	//   carrier
	// Type:
	//   string
	// Description:
	//   Carrier or ISP (e.g., “VERIZON”) using exchange curated string
	//   names which should be published to bidders a priori.
	Carrier string `json:"carrier,omitempty"`

	// Attribute:
	//   mccmnc
	// Type:
	//   string
	// Description:
	//   Mobile carrier as the concatenated MCC-MNC code
	//   (e.g., "310-005" identifies Verizon Wireless CDMA in the
	//   USA).
	//   Refer to https://en.wikipedia.org/wiki/Mobile_country_code
	//   for further examples. Note that the dash between the MCC
	//   and MNC parts is required to remove parsing ambiguity. The
	//   MCC-MNC values represent the SIM installed on the device
	//   and do not change when a device is roaming. Roaming may
	//   be inferred by a combination of the MCC-MNC, geo, IP and
	//   other data signals.
	MCCMNC string `json:"mccmnc,omitempty"`

	// Attribute:
	//   connectiontype
	// Type:
	//   integer
	// Description:
	//   Network connection type. Refer to List: Connection Types in
	//   AdCOM 1.0.
	ConnectionType int8 `json:"connectiontype,omitempty"`

	// Attribute:
	//   ifa
	// Type:
	//   string
	// Description:
	//   ID sanctioned for advertiser use in the clear (i.e., not hashed).
	IFA string `json:"ifa,omitempty"`

	// Attribute:
	//   didsha1
	// Type:
	//   string; DEPRECATED
	// Description:
	//   Hardware device ID (e.g., IMEI); hashed via SHA1.
	DIDSHA1 string `json:"didsha1,omitempty"`

	// Attribute:
	//   didmd5
	// Type:
	//   string; DEPRECATED
	// Description:
	//  Hardware device ID (e.g., IMEI); hashed via MD5.
	DIDMD5 string `json:"didmd5,omitempty"`

	// Attribute:
	//   dpidsha1
	// Type:
	//   string; DEPRECATED
	// Description:
	//   Platform device ID (e.g., Android ID); hashed via SHA1.
	DPIDSHA1 string `json:"dpidsha1,omitempty"`

	// Attribute:
	//   dpidmd5
	// Type:
	//   string; DEPRECATED
	// Description:
	//   Platform device ID (e.g., Android ID); hashed via MD5.
	DPIDMD5 string `json:"dpidmd5,omitempty"`

	// Attribute:
	//   macsha1
	// Type:
	//   string; DEPRECATED
	// Description:
	//   MAC address of the device; hashed via SHA1.
	MACSHA1 string `json:"macsha1,omitempty"`

	// Attribute:
	//   macmd5
	// Type:
	//   string; DEPRECATED
	// Description:
	//   MAC address of the device; hashed via MD5.
	MACMD5 string `json:"macmd5,omitempty"`

	// Attribute:
	//   ext
	// Type:
	//   object
	// Description:
	//   Placeholder for exchange-specific extensions to OpenRTB.
	Ext json.RawMessage `json:"ext,omitempty"`
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"net/http"

	"labix.org/v2/mgo/bson"
)

// UserInfo contains the relevant user models.
type UserInfo struct {
	ClientId, Username, SocialApiId string
	UserId, AccountId               bson.ObjectId
	Account                         *models.Account
	Impersonating                   bool
}

// fetchUseInfo fetches different user models and returns
// UserInfo. It first fetches `clientId` cookie and if it exists, it
// fetches JSession and other info.
func fetchUserInfo(w http.ResponseWriter, r *http.Request) (*UserInfo, error) {
	cookie, err := getCookie(w, r)
	if err != nil {
		return nil, err
	}

	clientId := cookie.Value
	session, err := fetchSession(clientId)
	if err != nil {
		expireClientId(w, r)
		return nil, err
	}

	username := session.Username

	account, err := fetchAccount(username)
	if err != nil {
		expireClientId(w, r)
		return nil, err
	}

	user, err := modelhelper.GetUser(username)
	if err != nil {
		return nil, err
	}

	userInfo := &UserInfo{
		ClientId:      clientId,
		Username:      username,
		SocialApiId:   account.SocialApiId,
		AccountId:     account.Id,
		UserId:        user.ObjectId,
		Account:       account,
		Impersonating: session.Impersonating,
	}

	return userInfo, nil
}

func fetchSession(clientId string) (*models.Session, error) {
	session, err := modelhelper.GetSession(clientId)
	if err != nil {
		return nil, err
	}

	username := session.Username
	if username == "" {
		err := errors.New(
			fmt.Sprintf("Username is empty for session: %s", clientId),
		)

		return nil, err
	}

	return session, nil
}

func fetchAccount(username string) (*models.Account, error) {
	account, err := modelhelper.GetAccount(username)
	if err != nil {
		return nil, err
	}

	if account.Type != "registered" {
		err := errors.New(fmt.Sprintf("Account is not register: %s", username))
		return nil, err
	}

	return account, nil
}
<commit_msg>go-webserver: better logs<commit_after>package main

import (
	"errors"
	"fmt"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"net/http"

	"labix.org/v2/mgo/bson"
)

// UserInfo contains the relevant user models and other info
// required to render loggedin page.
type UserInfo struct {
	ClientId, Username, SocialApiId string
	UserId, AccountId               bson.ObjectId
	Account                         *models.Account
	Impersonating                   bool
}

// fetchUseInfo fetches different user models and returns UserInfo.
// It first fetches `clientId` cookie and if it exists, it fetches
// JSession and other info.
func fetchUserInfo(w http.ResponseWriter, r *http.Request) (*UserInfo, error) {
	cookie, err := getCookie(w, r)
	if err != nil {
		return nil, err
	}

	clientId := cookie.Value
	session, err := fetchSession(clientId)
	if err != nil {
		expireClientId(w, r)
		return nil, err
	}

	username := session.Username

	account, err := fetchAccount(username)
	if err != nil {
		expireClientId(w, r)
		return nil, err
	}

	user, err := modelhelper.GetUser(username)
	if err != nil {
		return nil, err
	}

	userInfo := &UserInfo{
		ClientId:      clientId,
		Username:      username,
		SocialApiId:   account.SocialApiId,
		AccountId:     account.Id,
		UserId:        user.ObjectId,
		Account:       account,
		Impersonating: session.Impersonating,
	}

	return userInfo, nil
}

func fetchSession(clientId string) (*models.Session, error) {
	session, err := modelhelper.GetSession(clientId)
	if err != nil {
		return nil, err
	}

	username := session.Username
	if username == "" {
		err := errors.New(
			fmt.Sprintf("Username is empty for session: %s", clientId),
		)

		return nil, err
	}

	return session, nil
}

func fetchAccount(username string) (*models.Account, error) {
	account, err := modelhelper.GetAccount(username)
	if err != nil {
		return nil, err
	}

	if account.Type != "registered" {
		err := errors.New(fmt.Sprintf("Account is not register: %s", username))
		return nil, err
	}

	return account, nil
}
<|endoftext|>


<commit_before>package workergo

import (
	"sync"
)

// Dispatcher is the main code that runs and starts workers. Dispatcher is
// resposible of managing workers, dispatching them when needed
type Dispatcher struct {
	maxWorkers      int
	queueBufferSize int
	wg              *sync.WaitGroup
	wait            bool
	JobQueue        chan Job
	workerPool      chan chan Job
	workers         []Worker
	quit            chan bool
}

// NewDispatcher Creates a new dispatcher instance with given maximum number of
// workers
func NewDispatcher(maxWorkers int, queueBufferSize int) *Dispatcher {
	d := &Dispatcher{
		maxWorkers:      maxWorkers,
		queueBufferSize: queueBufferSize,
		workerPool:      make(chan chan Job, maxWorkers),
		wait:            false,
		quit:            make(chan bool),
		workers:         make([]Worker, maxWorkers),
	}

	if queueBufferSize != 0 {
		d.JobQueue = make(chan Job, queueBufferSize)
	} else {
		d.JobQueue = make(chan Job)
	}

	return d
}

// NewDispatcherWG Creates a new Dispatcher instance with given maximum number of
// workers and uses the given wait group to wait for workers to finish their jobs
func NewDispatcherWG(maxWorkers int, queueBufferSize int, exWg *sync.WaitGroup) *Dispatcher {
	d := NewDispatcher(maxWorkers, queueBufferSize)
	d.wg = exWg
	d.wait = true

	return d
}

// Run Starts the dispatcher
func (d *Dispatcher) Run() {
	// Start all workers
	for i := 0; i < d.maxWorkers; i++ {
		if d.wait {
			d.workers[i] = NewWorkerWG(d.workerPool, d.wg)
		} else {
			d.workers[i] = NewWorker(d.workerPool)
		}

		d.workers[i].Run()
	}

	go d.dispatch()
}

func (d *Dispatcher) dispatch() {
	defer d.shutdown()

	for {
		select {
		case job := <-d.JobQueue:
			// Get available workers channel
			workerJobChannel := <-d.workerPool

			// Send job to that channel
			workerJobChannel <- job
		case <-d.quit:
			return
		}
	}
}

// SubmitJob Submits given Job into job queue
func (d *Dispatcher) SubmitJob(job Job) {
	if d.wait {
		d.wg.Add(1)
	}

	// Send to job queue
	d.JobQueue <- job
}

// Stop Stops dispatcher
func (d *Dispatcher) Stop() {
	go func() {
		d.quit <- true
	}()
}

func (d *Dispatcher) shutdown() {
	for _, worker := range d.workers {
		for !worker.Stop() {

		}
	}

	close(d.workerPool)
	close(d.JobQueue)
}

// GetQueueSize returns current size of the job queue
func (d *Dispatcher) GetQueueSize() int {
	return len(d.JobQueue)
}
<commit_msg>Adding RunWithLimiter() method<commit_after>package workergo

import (
	"sync"
	"time"
)

// Dispatcher is the main code that runs and starts workers. Dispatcher is
// resposible of managing workers, dispatching them when needed
type Dispatcher struct {
	maxWorkers      int
	queueBufferSize int
	wg              *sync.WaitGroup
	wait            bool
	JobQueue        chan Job
	workerPool      chan chan Job
	workers         []Worker
	quit            chan bool
	limiter         <-chan time.Time
}

// NewDispatcher Creates a new dispatcher instance with given maximum number of
// workers
func NewDispatcher(maxWorkers int, queueBufferSize int) *Dispatcher {
	d := &Dispatcher{
		maxWorkers:      maxWorkers,
		queueBufferSize: queueBufferSize,
		workerPool:      make(chan chan Job, maxWorkers),
		wait:            false,
		quit:            make(chan bool),
		workers:         make([]Worker, maxWorkers),
	}

	if queueBufferSize != 0 {
		d.JobQueue = make(chan Job, queueBufferSize)
	} else {
		d.JobQueue = make(chan Job)
	}

	return d
}

// NewDispatcherWG Creates a new Dispatcher instance with given maximum number of
// workers and uses the given wait group to wait for workers to finish their jobs
func NewDispatcherWG(maxWorkers int, queueBufferSize int, exWg *sync.WaitGroup) *Dispatcher {
	d := NewDispatcher(maxWorkers, queueBufferSize)
	d.wg = exWg
	d.wait = true

	return d
}

// Run Starts the dispatcher
func (d *Dispatcher) Run() {
	// Start all workers
	for i := 0; i < d.maxWorkers; i++ {
		if d.wait {
			d.workers[i] = NewWorkerWG(d.workerPool, d.wg)
		} else {
			d.workers[i] = NewWorker(d.workerPool)
		}

		d.workers[i].Run()
	}

	go d.dispatch()
}

// RunWithLimiter Adds delays between jobs
func (d *Dispatcher) RunWithLimiter(limiterGap time.Duration) {
	d.limiter = time.Tick(limiterGap)
	d.Run()
}

func (d *Dispatcher) dispatch() {
	defer d.shutdown()

	for {
		select {
		case job := <-d.JobQueue:
			// Get available workers channel
			workerJobChannel := <-d.workerPool

			if d.limiter != nil {
				<-d.limiter
			}

			// Send job to that channel
			workerJobChannel <- job
		case <-d.quit:
			return
		}
	}
}

// SubmitJob Submits given Job into job queue
func (d *Dispatcher) SubmitJob(job Job) {
	if d.wait {
		d.wg.Add(1)
	}

	// Send to job queue
	d.JobQueue <- job
}

// Stop Stops dispatcher
func (d *Dispatcher) Stop() {
	go func() {
		d.quit <- true
	}()
}

func (d *Dispatcher) shutdown() {
	for _, worker := range d.workers {
		for !worker.Stop() {

		}
	}

	close(d.workerPool)
	close(d.JobQueue)
}

// GetQueueSize returns current size of the job queue
func (d *Dispatcher) GetQueueSize() int {
	return len(d.JobQueue)
}
<|endoftext|>


<commit_before>package version

// VERSION ...
const VERSION = "1.1.45"
<commit_msg>v1.1.46<commit_after>package version

// VERSION ...
const VERSION = "1.1.46"
<|endoftext|>


<commit_before>package controllers

import "github.com/revel/revel"

type App struct {
	*revel.Controller
}

func (c App) Index() revel.Result {
	return c.Render()
}
<commit_msg>add cookies sessions<commit_after>package controllers

import "github.com/revel/revel"

type App struct {
	*revel.Controller
}

type Session map[string]string

func (c App) Index() revel.Result {
	return c.Render()
}
<|endoftext|>


<commit_before>package reflect

import (
	. "github.com/anaminus/rbxmk"
	"github.com/anaminus/rbxmk/rtypes"
	"github.com/robloxapi/types"
)

func PropertyDesc() Type {
	return Type{
		Name:     "PropertyDesc",
		PushTo:   PushTypeTo,
		PullFrom: PullTypeFrom,
		Members: Members{
			"Name": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(types.String(desc.Name))
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.Name = string(s.Pull(3, "string").(types.String))
				},
			},
			"ValueType": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(rtypes.TypeDesc{Embedded: desc.ValueType})
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.ValueType = s.Pull(3, "TypeDesc").(rtypes.TypeDesc).Embedded
				},
			},
			"ReadSecurity": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(types.String(desc.ReadSecurity))
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.ReadSecurity = string(s.Pull(3, "string").(types.String))
				},
			},
			"WriteSecurity": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(types.String(desc.WriteSecurity))
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.WriteSecurity = string(s.Pull(3, "string").(types.String))
				},
			},
			"CanLoad": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(types.Bool(desc.CanLoad))
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.CanLoad = bool(s.Pull(3, "string").(types.Bool))
				},
			},
			"CanSave": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(types.Bool(desc.CanSave))
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.CanSave = bool(s.Pull(3, "string").(types.Bool))
				},
			},
			"Tag": Member{Method: true, Get: func(s State, v types.Value) int {
				desc := v.(rtypes.PropertyDesc)
				tag := string(s.Pull(2, "string").(types.String))
				return s.Push(types.Bool(desc.GetTag(tag)))
			}},
			"Tags": Member{Method: true, Get: func(s State, v types.Value) int {
				desc := v.(rtypes.PropertyDesc)
				tags := desc.GetTags()
				array := make(rtypes.Array, len(tags))
				for i, tag := range tags {
					array[i] = types.String(tag)
				}
				return s.Push(array)
			}},
			"SetTag": Member{Method: true, Get: func(s State, v types.Value) int {
				desc := v.(rtypes.PropertyDesc)
				tags := make([]string, s.Count()-1)
				for i := 2; i <= s.Count(); i++ {
					tags[i-2] = string(s.Pull(i, "string").(types.String))
				}
				desc.SetTag(tags...)
				return 0
			}},
			"UnsetTag": Member{Method: true, Get: func(s State, v types.Value) int {
				desc := v.(rtypes.PropertyDesc)
				tags := make([]string, s.Count()-1)
				for i := 2; i <= s.Count(); i++ {
					tags[i-2] = string(s.Pull(i, "string").(types.String))
				}
				desc.UnsetTag(tags...)
				return 0
			}},
		},
	}
}
<commit_msg>Fix pulled type of CanLoad/CanSave.<commit_after>package reflect

import (
	. "github.com/anaminus/rbxmk"
	"github.com/anaminus/rbxmk/rtypes"
	"github.com/robloxapi/types"
)

func PropertyDesc() Type {
	return Type{
		Name:     "PropertyDesc",
		PushTo:   PushTypeTo,
		PullFrom: PullTypeFrom,
		Members: Members{
			"Name": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(types.String(desc.Name))
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.Name = string(s.Pull(3, "string").(types.String))
				},
			},
			"ValueType": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(rtypes.TypeDesc{Embedded: desc.ValueType})
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.ValueType = s.Pull(3, "TypeDesc").(rtypes.TypeDesc).Embedded
				},
			},
			"ReadSecurity": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(types.String(desc.ReadSecurity))
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.ReadSecurity = string(s.Pull(3, "string").(types.String))
				},
			},
			"WriteSecurity": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(types.String(desc.WriteSecurity))
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.WriteSecurity = string(s.Pull(3, "string").(types.String))
				},
			},
			"CanLoad": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(types.Bool(desc.CanLoad))
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.CanLoad = bool(s.Pull(3, "bool").(types.Bool))
				},
			},
			"CanSave": Member{
				Get: func(s State, v types.Value) int {
					desc := v.(rtypes.PropertyDesc)
					return s.Push(types.Bool(desc.CanSave))
				},
				Set: func(s State, v types.Value) {
					desc := v.(rtypes.PropertyDesc)
					desc.CanSave = bool(s.Pull(3, "bool").(types.Bool))
				},
			},
			"Tag": Member{Method: true, Get: func(s State, v types.Value) int {
				desc := v.(rtypes.PropertyDesc)
				tag := string(s.Pull(2, "string").(types.String))
				return s.Push(types.Bool(desc.GetTag(tag)))
			}},
			"Tags": Member{Method: true, Get: func(s State, v types.Value) int {
				desc := v.(rtypes.PropertyDesc)
				tags := desc.GetTags()
				array := make(rtypes.Array, len(tags))
				for i, tag := range tags {
					array[i] = types.String(tag)
				}
				return s.Push(array)
			}},
			"SetTag": Member{Method: true, Get: func(s State, v types.Value) int {
				desc := v.(rtypes.PropertyDesc)
				tags := make([]string, s.Count()-1)
				for i := 2; i <= s.Count(); i++ {
					tags[i-2] = string(s.Pull(i, "string").(types.String))
				}
				desc.SetTag(tags...)
				return 0
			}},
			"UnsetTag": Member{Method: true, Get: func(s State, v types.Value) int {
				desc := v.(rtypes.PropertyDesc)
				tags := make([]string, s.Count()-1)
				for i := 2; i <= s.Count(); i++ {
					tags[i-2] = string(s.Pull(i, "string").(types.String))
				}
				desc.UnsetTag(tags...)
				return 0
			}},
		},
	}
}
<|endoftext|>


<commit_before>// Command line app to process fiddles in bulk.
//
// Example:
//  fiddlecli --input demo/testbulk.json --output /tmp/output.json
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"sync"

	"golang.org/x/sync/errgroup"

	"go.skia.org/infra/fiddle/go/types"
	"go.skia.org/infra/go/httputils"
)

// flags
var (
	domain = flag.String("domain", "https://fiddle.skia.org", "Where to send the JSON request.")
	input  = flag.String("input", "", "The name of the file to read the JSON from.")
	output = flag.String("output", "", "The name of the file to write the JSON results to.")
	procs  = flag.Int("procs", 4, "The number of parallel requests to make to the fiddle server.")
	quiet  = flag.Bool("quiet", false, "Run without a progress bar.")
	force  = flag.Bool("force", false, "Force a compile and run for each fiddle, don't take the fast path.")
)

// chanRequest is sent to each worker in the pool.
type chanRequest struct {
	id  string
	req *types.FiddleContext
}

func main() {
	// Check flags.
	flag.Parse()
	if *input == "" {
		flag.Usage()
		log.Fatalf("--input is a required flag.")
	}
	if *output == "" {
		flag.Usage()
		log.Fatalf("--output is a required flag.")
	}

	// Read the source JSON file.
	b, err := ioutil.ReadFile(*input)
	if err != nil {
		log.Fatalf("Failed to read %s: %s", *input, err)
	}
	requests := types.BulkRequest{}
	if err := json.Unmarshal(b, &requests); err != nil {
		log.Fatalf("%s does not contain valid JSON: %s", *input, err)
	}

	g := errgroup.Group{}
	requestsCh := make(chan chanRequest, len(requests))

	// mutex protects response.
	mutex := sync.Mutex{}
	response := types.BulkResponse{}

	// Spin up workers.
	for i := 0; i < *procs; i++ {
		g.Go(func() error {
			c := httputils.NewTimeoutClient()
			for req := range requestsCh {
				if !*quiet {
					fmt.Print(".")
				}
				if *force {
					req.req.Fast = false
				}
				// POST to fiddle.
				b, err = json.Marshal(req.req)
				if err != nil {
					return fmt.Errorf("Failed to encode an individual request: %s", err)
				}
				resp, err := c.Post(*domain+"/_/run", "application/json", bytes.NewReader(b))
				if err != nil || resp.StatusCode != 200 {
					return fmt.Errorf("Failed to make request: %s", err)
				}

				// Decode response and add to all responses.
				runResults := types.RunResults{}
				if err := json.NewDecoder(resp.Body).Decode(&runResults); err != nil {
					return fmt.Errorf("Failed to read response: %s", err)
				}
				if resp.StatusCode != 200 {
					return fmt.Errorf("Request failed with %d: %s", resp.StatusCode, string(b))
				}
				mutex.Lock()
				response[req.id] = &runResults
				mutex.Unlock()
			}
			return nil
		})
	}

	// Loop over each entry and queue them up for the workers.
	for id, req := range requests {
		requestsCh <- chanRequest{
			id:  id,
			req: req,
		}
	}
	close(requestsCh)

	// Wait for the workers to finish.
	if err := g.Wait(); err != nil {
		log.Fatalf("Failed to complete all requests: %s", err)
	}

	if !*quiet {
		fmt.Print("\n")
	}
	b, err = json.MarshalIndent(response, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode response file: %s", err)
	}
	if err := ioutil.WriteFile(*output, b, 0600); err != nil {
		log.Fatalf("Failed to write response file: %s", err)
	}
}
<commit_msg>check local hash before calling server<commit_after>// Command line app to process fiddles in bulk.
//
// Example:
//  fiddlecli --input demo/testbulk.json --output /tmp/output.json
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"sync"

	"golang.org/x/sync/errgroup"

	"go.skia.org/infra/fiddle/go/types"
	"go.skia.org/infra/go/httputils"
)

// flags
var (
	domain = flag.String("domain", "https://fiddle.skia.org", "Where to send the JSON request.")
	input  = flag.String("input", "", "The name of the file to read the JSON from.")
	output = flag.String("output", "", "The name of the file to write the JSON results to.")
	procs  = flag.Int("procs", 4, "The number of parallel requests to make to the fiddle server.")
	quiet  = flag.Bool("quiet", false, "Run without a progress bar.")
	force  = flag.Bool("force", false, "Force a compile and run for each fiddle, don't take the fast path.")
)

// chanRequest is sent to each worker in the pool.
type chanRequest struct {
	id  string
	req *types.FiddleContext
}

func main() {
	// Check flags.
	flag.Parse()
	if *input == "" {
		flag.Usage()
		log.Fatalf("--input is a required flag.")
	}
	if *output == "" {
		flag.Usage()
		log.Fatalf("--output is a required flag.")
	}

	// Read the source JSON file.
	b, err := ioutil.ReadFile(*input)
	if err != nil {
		log.Fatalf("Failed to read %s: %s", *input, err)
	}
	requests := types.BulkRequest{}
	if err := json.Unmarshal(b, &requests); err != nil {
		log.Fatalf("%s does not contain valid JSON: %s", *input, err)
	}
	lastWritten := types.BulkResponse{}
	b, err = ioutil.ReadFile(*output)
	if err == nil {
		if err := json.Unmarshal(b, &lastWritten); err != nil {
			lastWritten = nil
		}
	}
	g := errgroup.Group{}
	requestsCh := make(chan chanRequest, len(requests))

	// mutex protects response.
	mutex := sync.Mutex{}
	response := types.BulkResponse{}

	// Spin up workers.
	for i := 0; i < *procs; i++ {
		g.Go(func() error {
			c := httputils.NewTimeoutClient()
			for req := range requestsCh {
				if !*quiet {
					fmt.Print(".")
				}
				if *force {
					req.req.Fast = false
				} else if lastWritten != nil {
					fiddleHash, err := req.req.Options.ComputeHash(req.req.Code)
					if err == nil {
						if lastWritten[req.id] != nil {
							if lastWritten[req.id].FiddleHash == fiddleHash {
								mutex.Lock()
								response[req.id] = lastWritten[req.id]
								mutex.Unlock()
								continue
							}
						}
					}
				}
				// POST to fiddle.
				b, err = json.Marshal(req.req)
				if err != nil {
					return fmt.Errorf("Failed to encode an individual request: %s", err)
				}
				resp, err := c.Post(*domain+"/_/run", "application/json", bytes.NewReader(b))
				if err != nil || resp.StatusCode != 200 {
					return fmt.Errorf("Failed to make request: %s", err)
				}

				// Decode response and add to all responses.
				runResults := types.RunResults{}
				if err := json.NewDecoder(resp.Body).Decode(&runResults); err != nil {
					return fmt.Errorf("Failed to read response: %s", err)
				}
				if resp.StatusCode != 200 {
					return fmt.Errorf("Request failed with %d: %s", resp.StatusCode, string(b))
				}
				mutex.Lock()
				response[req.id] = &runResults
				mutex.Unlock()
			}
			return nil
		})
	}

	// Loop over each entry and queue them up for the workers.
	for id, req := range requests {
		requestsCh <- chanRequest{
			id:  id,
			req: req,
		}
	}
	close(requestsCh)

	// Wait for the workers to finish.
	if err := g.Wait(); err != nil {
		log.Fatalf("Failed to complete all requests: %s", err)
	}

	if !*quiet {
		fmt.Print("\n")
	}
	b, err = json.MarshalIndent(response, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode response file: %s", err)
	}
	if err := ioutil.WriteFile(*output, b, 0600); err != nil {
		log.Fatalf("Failed to write response file: %s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
//	"fmt"
	"encoding/binary"
	"log"
	"net"
	"os"
	"time"
	"github.com/miekg/dns"
	"github.com/miekg/pcap"
)

// We store the configuration of our local resolver in a global
// variable for convenience.
var (
	resolv_conf	*dns.ClientConfig
)

// We have a goroutine to act as a stub resolver, and use this
// structure to send the question in and get the results out.
type stub_resolve_info struct {
	ownername	string
	rtype		uint16
	answer		*dns.Msg
}

// A goroutine which performs stub lookups from a queue, writing
// the results to another queue.
func stub_resolve(questions <-chan stub_resolve_info,
                  results chan<- stub_resolve_info) {
	// make a client for our lookups
	dnsClient := new(dns.Client)
	dnsClient.Net = "tcp"
	// read each question on our channel
	for question := range questions {
		// build our answer
		var result stub_resolve_info = question
		result.answer = nil
		// make a DNS query based on our question
		query := new(dns.Msg)
		query.RecursionDesired = true
		query.SetQuestion(question.ownername, question.rtype)
		// check each resolver in turn
		for _, server := range resolv_conf.Servers {
			resolver := server + ":53"
			r , _, err := dnsClient.Exchange(query, resolver)
			// if we got an answer, use that and stop trying
			if (err == nil) && (r != nil) && (r.Rcode == dns.RcodeSuccess) {
				result.answer = r
				break
			}
		}
		// send back our answer (might be nil)
		results <- result
	}
}

// Lookup all of the IP addresses associated with the root name servers.
// Return two maps based on the results found, which have the keys of
// the binary values of the IPv4 and IPv6 addresses. (It's a bit clumsy,
// but it allows us to do quick and easy lookups of the addresses in the
// pcap later.)
func get_root_server_addresses() (map[[4]byte]bool, map[[16]byte]bool) {
	// look up the NS of the IANA root
	root_client := new(dns.Client)
	root_client.Net = "tcp"
	ns_query := new(dns.Msg)
	ns_query.SetQuestion(".", dns.TypeNS)
	// TODO: avoid hard-coding a particular root server here
	ns_response , _, err := root_client.Exchange(ns_query,
                                                     "k.root-servers.net:53")
	if err != nil {
		log.Fatal("Error looking up root name servers")
	}
	var root_servers []string
	for _, root_server := range ns_response.Answer {
		switch root_server.(type) {
		case *dns.NS:
			ns := root_server.(*dns.NS).Ns
			root_servers = append(root_servers, ns)
		}
	}
	// look up the addresses of the root servers
	questions := make(chan stub_resolve_info, 16)
	results := make(chan stub_resolve_info, len(root_servers)*2)
	for i := 0; i<8; i++ {
		go stub_resolve(questions, results)
	}
	for _, ns := range root_servers {
		info := new(stub_resolve_info)
		info.ownername = ns
		info.rtype = dns.TypeAAAA
		questions <- *info
		info = new(stub_resolve_info)
		info.ownername = ns
		info.rtype = dns.TypeA
		questions <- *info
	}
	root_addresses4 := make(map[[4]byte]bool)
	root_addresses6 := make(map[[16]byte]bool)
	for i := 0; i<len(root_servers)*2; i++ {
		response := <-results
		if response.answer == nil {
			log.Fatal("Error looking up root server %s",
			          response.ownername)
		}
		for _, root_address := range response.answer.Answer {
			switch root_address.(type) {
			case *dns.AAAA:
				aaaa_s := root_address.(*dns.AAAA).AAAA.String()
				var aaaa [16]byte
				copy(aaaa[:], net.ParseIP(aaaa_s)[0:16])
				root_addresses6[aaaa] = true
			case *dns.A:
				a_s := root_address.(*dns.A).A.String()
				var a [4]byte
				copy(a[:], net.ParseIP(a_s)[0:4])
				root_addresses4[a] = true
			}
		}
	}
	close(questions)
	return root_addresses4, root_addresses6
}

func ymmv_write(ip_family int, addr []byte, query dns.Msg,
		answer_time time.Time, answer dns.Msg) {
	// output magic value
	_, err := os.Stdout.Write([]byte("ymmv"))
	if err != nil {
		log.Fatal(err)
	}

	// output address family
	if ip_family == 4 {
		_, err = os.Stdout.Write([]byte("4"))
	} else if ip_family == 6 {
		_, err = os.Stdout.Write([]byte("6"))
	} else {
		log.Fatal("Unknown ip_family %d\n", ip_family)
	}
	if err != nil {
		log.Fatal(err)
	}

	// output (U)DP or (T)CP
	_, err = os.Stdout.Write([]byte("u"))	// only support UDP for now...
	if err != nil {
		log.Fatal(err)
	}

	// output actual address
	_, err = os.Stdout.Write(addr)
	if err != nil {
		log.Fatal(err)
	}

	// output when the query happened (we don't know, so use 0)
	_, err = os.Stdout.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0})

	// write the byte count of our query
	query_bytes, err := query.Pack()
	if err != nil {
		log.Fatal(err)
	}
	query_len := uint16(len(query_bytes))
	err = binary.Write(os.Stdout, binary.BigEndian, query_len)
	if err != nil {
		log.Fatal(err)
	}

	// write our query
	_, err = os.Stdout.Write(query_bytes)
	if err != nil {
		log.Fatal(err)
	}

	// output when the answer arrived
	seconds := uint32(answer_time.Second())
	err = binary.Write(os.Stdout, binary.BigEndian, seconds)
	if err != nil {
		log.Fatal(err)
	}
	nanoseconds := uint32(answer_time.Nanosecond())
	err = binary.Write(os.Stdout, binary.BigEndian, nanoseconds)
	if err != nil {
		log.Fatal(err)
	}

	// write the byte count of our answer
	answer_bytes, err := answer.Pack()
	if err != nil {
		log.Fatal(err)
	}
	answer_len := uint16(len(answer_bytes))
	err = binary.Write(os.Stdout, binary.BigEndian, answer_len)
	if err != nil {
		log.Fatal(err)
	}

	// write our query
	_, err = os.Stdout.Write(answer_bytes)
	if err != nil {
		log.Fatal(err)
	}

	// XXX: do we need to flush?
}

// Look in the named file and find any packets that are from our root
// servers on port 53.
func pcap2ymmv(fname string,
               root_addresses4 map[[4]byte]bool,
               root_addresses6 map[[16]byte]bool) {
	// open our pcap file
	file, err := os.Open(fname)
	if err != nil {
		log.Fatal(err)
	}
	pcap_file, err := pcap.NewReader(file)
	if err != nil {
		log.Fatal(err)
	}

	for {
		// reach each packet
		pkt := pcap_file.Next()
		if pkt == nil {
			break
		}
		pkt.Decode()

		// parse each header so we can see if we want this packet
		ip_family := 0
		var ip_addr []byte
		valid_udp := false
		for _, hdr := range pkt.Headers {
			switch hdr.(type) {
			case *pcap.Iphdr:
				// check that the packet comes from one
				// of the addresses that we are looking for
				iphdr := hdr.(*pcap.Iphdr)
				var addr [4]byte
				copy(addr[:], iphdr.SrcIp[0:4])
				_, found :=  root_addresses4[addr]
				if found {
					ip_family = 4
					ip_addr = make([]byte, 4)
					copy(ip_addr[:], addr[0:4])
				}
			case *pcap.Ip6hdr:
				iphdr := hdr.(*pcap.Ip6hdr)
				var addr [16]byte
				copy(addr[:], iphdr.SrcIp[0:16])
				_, found :=  root_addresses6[addr]
				if found {
					ip_family = 6
					ip_addr = make([]byte, 16)
					copy(ip_addr[:], addr[0:16])
				}
			case *pcap.Udphdr:
				udphdr := hdr.(*pcap.Udphdr)
				if udphdr.SrcPort == 53 {
					valid_udp = true
				}
			}
		}

		if (ip_family != 0) && valid_udp {
			answer := new(dns.Msg)
			answer.Unpack(pkt.Payload)
			answer.Id = 0
			query := answer.Copy()
			query.Response = false
			query.Authoritative = false
			query.Truncated = false
			query.AuthenticatedData = true
			query.CheckingDisabled = false
			query.Rcode = 0
			query.Answer = nil
			query.Ns = nil
			old_extra := query.Extra
			query.Extra = nil
			// add our opt section back - probably not really
			// what we want, but what else can we do?
			if old_extra != nil {
				for _, extra := range old_extra {
					switch extra.(type) {
					case *dns.OPT:
						opt := extra.(*dns.OPT)
						query.Extra = []dns.RR{opt}
					}
				}
			}
			ymmv_write(ip_family, ip_addr,
				   *query, pkt.Time, *answer)
//			fmt.Printf("%s\n", answer)
//			fmt.Printf("%s\n", query)
		}
	}
	file.Close()
}

// Main function.
func main() {
	// initialize our stub resolver
	var ( err error )
	resolv_conf, err = dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil {
		log.Fatal(err)
	}

	// get root server addresses
	root_addresses4, root_addresses6 := get_root_server_addresses()

	// process each argument as a pcap file
	for _, fname := range os.Args[1:] {
		pcap2ymmv(fname, root_addresses4, root_addresses6)
	}
}
<commit_msg>Add support for "-" as input filename<commit_after>package main

import (
//	"fmt"
	"encoding/binary"
	"log"
	"net"
	"os"
	"time"
	"github.com/miekg/dns"
	"github.com/miekg/pcap"
)

// We store the configuration of our local resolver in a global
// variable for convenience.
var (
	resolv_conf	*dns.ClientConfig
)

// We have a goroutine to act as a stub resolver, and use this
// structure to send the question in and get the results out.
type stub_resolve_info struct {
	ownername	string
	rtype		uint16
	answer		*dns.Msg
}

// A goroutine which performs stub lookups from a queue, writing
// the results to another queue.
func stub_resolve(questions <-chan stub_resolve_info,
                  results chan<- stub_resolve_info) {
	// make a client for our lookups
	dnsClient := new(dns.Client)
	dnsClient.Net = "tcp"
	// read each question on our channel
	for question := range questions {
		// build our answer
		var result stub_resolve_info = question
		result.answer = nil
		// make a DNS query based on our question
		query := new(dns.Msg)
		query.RecursionDesired = true
		query.SetQuestion(question.ownername, question.rtype)
		// check each resolver in turn
		for _, server := range resolv_conf.Servers {
			resolver := server + ":53"
			r , _, err := dnsClient.Exchange(query, resolver)
			// if we got an answer, use that and stop trying
			if (err == nil) && (r != nil) && (r.Rcode == dns.RcodeSuccess) {
				result.answer = r
				break
			}
		}
		// send back our answer (might be nil)
		results <- result
	}
}

// Lookup all of the IP addresses associated with the root name servers.
// Return two maps based on the results found, which have the keys of
// the binary values of the IPv4 and IPv6 addresses. (It's a bit clumsy,
// but it allows us to do quick and easy lookups of the addresses in the
// pcap later.)
func get_root_server_addresses() (map[[4]byte]bool, map[[16]byte]bool) {
	// look up the NS of the IANA root
	root_client := new(dns.Client)
	root_client.Net = "tcp"
	ns_query := new(dns.Msg)
	ns_query.SetQuestion(".", dns.TypeNS)
	// TODO: avoid hard-coding a particular root server here
	ns_response , _, err := root_client.Exchange(ns_query,
                                                     "k.root-servers.net:53")
	if err != nil {
		log.Fatal("Error looking up root name servers")
	}
	var root_servers []string
	for _, root_server := range ns_response.Answer {
		switch root_server.(type) {
		case *dns.NS:
			ns := root_server.(*dns.NS).Ns
			root_servers = append(root_servers, ns)
		}
	}
	// look up the addresses of the root servers
	questions := make(chan stub_resolve_info, 16)
	results := make(chan stub_resolve_info, len(root_servers)*2)
	for i := 0; i<8; i++ {
		go stub_resolve(questions, results)
	}
	for _, ns := range root_servers {
		info := new(stub_resolve_info)
		info.ownername = ns
		info.rtype = dns.TypeAAAA
		questions <- *info
		info = new(stub_resolve_info)
		info.ownername = ns
		info.rtype = dns.TypeA
		questions <- *info
	}
	root_addresses4 := make(map[[4]byte]bool)
	root_addresses6 := make(map[[16]byte]bool)
	for i := 0; i<len(root_servers)*2; i++ {
		response := <-results
		if response.answer == nil {
			log.Fatal("Error looking up root server %s",
			          response.ownername)
		}
		for _, root_address := range response.answer.Answer {
			switch root_address.(type) {
			case *dns.AAAA:
				aaaa_s := root_address.(*dns.AAAA).AAAA.String()
				var aaaa [16]byte
				copy(aaaa[:], net.ParseIP(aaaa_s)[0:16])
				root_addresses6[aaaa] = true
			case *dns.A:
				a_s := root_address.(*dns.A).A.String()
				var a [4]byte
				copy(a[:], net.ParseIP(a_s)[0:4])
				root_addresses4[a] = true
			}
		}
	}
	close(questions)
	return root_addresses4, root_addresses6
}

func ymmv_write(ip_family int, addr []byte, query dns.Msg,
		answer_time time.Time, answer dns.Msg) {
	// output magic value
	_, err := os.Stdout.Write([]byte("ymmv"))
	if err != nil {
		log.Fatal(err)
	}

	// output address family
	if ip_family == 4 {
		_, err = os.Stdout.Write([]byte("4"))
	} else if ip_family == 6 {
		_, err = os.Stdout.Write([]byte("6"))
	} else {
		log.Fatal("Unknown ip_family %d\n", ip_family)
	}
	if err != nil {
		log.Fatal(err)
	}

	// output (U)DP or (T)CP
	_, err = os.Stdout.Write([]byte("u"))	// only support UDP for now...
	if err != nil {
		log.Fatal(err)
	}

	// output actual address
	_, err = os.Stdout.Write(addr)
	if err != nil {
		log.Fatal(err)
	}

	// output when the query happened (we don't know, so use 0)
	_, err = os.Stdout.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0})

	// write the byte count of our query
	query_bytes, err := query.Pack()
	if err != nil {
		log.Fatal(err)
	}
	query_len := uint16(len(query_bytes))
	err = binary.Write(os.Stdout, binary.BigEndian, query_len)
	if err != nil {
		log.Fatal(err)
	}

	// write our query
	_, err = os.Stdout.Write(query_bytes)
	if err != nil {
		log.Fatal(err)
	}

	// output when the answer arrived
	seconds := uint32(answer_time.Second())
	err = binary.Write(os.Stdout, binary.BigEndian, seconds)
	if err != nil {
		log.Fatal(err)
	}
	nanoseconds := uint32(answer_time.Nanosecond())
	err = binary.Write(os.Stdout, binary.BigEndian, nanoseconds)
	if err != nil {
		log.Fatal(err)
	}

	// write the byte count of our answer
	answer_bytes, err := answer.Pack()
	if err != nil {
		log.Fatal(err)
	}
	answer_len := uint16(len(answer_bytes))
	err = binary.Write(os.Stdout, binary.BigEndian, answer_len)
	if err != nil {
		log.Fatal(err)
	}

	// write our query
	_, err = os.Stdout.Write(answer_bytes)
	if err != nil {
		log.Fatal(err)
	}

	// XXX: do we need to flush?
}

// Look in the named file and find any packets that are from our root
// servers on port 53.
func pcap2ymmv(fname string,
               root_addresses4 map[[4]byte]bool,
               root_addresses6 map[[16]byte]bool) {
	// open our pcap file
	var file *os.File
	if fname == "-" {
		file = os.Stdin
	} else {
		named_file, err := os.Open(fname)
		if err != nil {
			log.Fatal(err)
		}
		file = named_file
	}
	pcap_file, err := pcap.NewReader(file)
	if err != nil {
		log.Fatal(err)
	}

	for {
		// reach each packet
		pkt := pcap_file.Next()
		if pkt == nil {
			break
		}
		pkt.Decode()

		// parse each header so we can see if we want this packet
		ip_family := 0
		var ip_addr []byte
		valid_udp := false
		for _, hdr := range pkt.Headers {
			switch hdr.(type) {
			case *pcap.Iphdr:
				// check that the packet comes from one
				// of the addresses that we are looking for
				iphdr := hdr.(*pcap.Iphdr)
				var addr [4]byte
				copy(addr[:], iphdr.SrcIp[0:4])
				_, found :=  root_addresses4[addr]
				if found {
					ip_family = 4
					ip_addr = make([]byte, 4)
					copy(ip_addr[:], addr[0:4])
				}
			case *pcap.Ip6hdr:
				iphdr := hdr.(*pcap.Ip6hdr)
				var addr [16]byte
				copy(addr[:], iphdr.SrcIp[0:16])
				_, found :=  root_addresses6[addr]
				if found {
					ip_family = 6
					ip_addr = make([]byte, 16)
					copy(ip_addr[:], addr[0:16])
				}
			case *pcap.Udphdr:
				udphdr := hdr.(*pcap.Udphdr)
				if udphdr.SrcPort == 53 {
					valid_udp = true
				}
			}
		}

		if (ip_family != 0) && valid_udp {
			answer := new(dns.Msg)
			answer.Unpack(pkt.Payload)
			answer.Id = 0
			query := answer.Copy()
			query.Response = false
			query.Authoritative = false
			query.Truncated = false
			query.AuthenticatedData = true
			query.CheckingDisabled = false
			query.Rcode = 0
			query.Answer = nil
			query.Ns = nil
			old_extra := query.Extra
			query.Extra = nil
			// add our opt section back - probably not really
			// what we want, but what else can we do?
			if old_extra != nil {
				for _, extra := range old_extra {
					switch extra.(type) {
					case *dns.OPT:
						opt := extra.(*dns.OPT)
						query.Extra = []dns.RR{opt}
					}
				}
			}
			ymmv_write(ip_family, ip_addr,
				   *query, pkt.Time, *answer)
		}
	}
	file.Close()
}

// Main function.
func main() {
	// initialize our stub resolver
	var ( err error )
	resolv_conf, err = dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil {
		log.Fatal(err)
	}

	// get root server addresses
	root_addresses4, root_addresses6 := get_root_server_addresses()

	// process each argument as a pcap file
	for _, fname := range os.Args[1:] {
		pcap2ymmv(fname, root_addresses4, root_addresses6)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018-2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"
	"time"

	"github.com/cilium/cilium/test/config"
	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/gomega"
)

var longTimeout = 10 * time.Minute

// ExpectKubeDNSReady is a wrapper around helpers/WaitKubeDNS. It asserts that
// the error returned by that function is nil.
func ExpectKubeDNSReady(vm *helpers.Kubectl) {
	By("Waiting for kube-dns to be ready")
	err := vm.WaitKubeDNS()
	ExpectWithOffset(1, err).Should(BeNil(), "kube-dns was not able to get into ready state")

	By("Running kube-dns preflight check")
	err = vm.KubeDNSPreFlightCheck()
	ExpectWithOffset(1, err).Should(BeNil(), "kube-dns service not ready")
}

// ExpectCiliumReady is a wrapper around helpers/WaitForPods. It asserts that
// the error returned by that function is nil.
func ExpectCiliumReady(vm *helpers.Kubectl) {
	err := vm.WaitForCiliumReadiness()
	Expect(err).To(BeNil(), "Timeout while waiting for Cilium to become ready")

	err = vm.CiliumPreFlightCheck()
	ExpectWithOffset(1, err).Should(BeNil(), "cilium pre-flight checks failed")
}

// ExpectCiliumOperatorReady is a wrapper around helpers/WaitForPods. It asserts that
// the error returned by that function is nil.
func ExpectCiliumOperatorReady(vm *helpers.Kubectl) {
	By("Waiting for cilium-operator to be ready")
	err := vm.WaitforPods(helpers.CiliumNamespace, "-l name=cilium-operator", longTimeout)
	ExpectWithOffset(1, err).Should(BeNil(), "Cilium operator was not able to get into ready state")
}

// ExpectHubbleCLIReady is a wrapper around helpers/WaitForPods. It asserts
// that the error returned by that function is nil.
func ExpectHubbleCLIReady(vm *helpers.Kubectl, ns string) {
	By("Waiting for hubble-cli to be ready")
	err := vm.WaitforPods(ns, "-l k8s-app=hubble-cli", longTimeout)
	ExpectWithOffset(1, err).Should(BeNil(), "hubble-cli was not able to get into ready state")
}

// ExpectHubbleRelayReady is a wrapper around helpers/WaitForPods. It asserts
// that the error returned by that function is nil.
func ExpectHubbleRelayReady(vm *helpers.Kubectl, ns string) {
	By("Waiting for hubble-relay to be ready")
	err := vm.WaitforPods(ns, "-l k8s-app=hubble-relay", longTimeout)
	ExpectWithOffset(1, err).Should(BeNil(), "hubble-relay was not able to get into ready state")
}

// ExpectAllPodsTerminated is a wrapper around helpers/WaitCleanAllTerminatingPods.
// It asserts that the error returned by that function is nil.
func ExpectAllPodsTerminated(vm *helpers.Kubectl) {
	err := vm.WaitCleanAllTerminatingPods(helpers.HelperTimeout)
	ExpectWithOffset(1, err).To(BeNil(), "terminating containers are not deleted after timeout")
}

// ExpectAllPodsInNsTerminated is a wrapper around helpers/WaitCleanAllTerminatingPods.
// It asserts that the error returned by that function is nil.
func ExpectAllPodsInNsTerminated(vm *helpers.Kubectl, ns string) {
	err := vm.WaitCleanAllTerminatingPodsInNs(ns, helpers.HelperTimeout)
	ExpectWithOffset(1, err).To(BeNil(), "terminating containers are not deleted after timeout")
}

// ExpectCiliumPreFlightInstallReady is a wrapper around helpers/WaitForNPods.
// It asserts the error returned by that function is nil.
func ExpectCiliumPreFlightInstallReady(vm *helpers.Kubectl) {
	By("Waiting for all cilium pre-flight pods to be ready")

	err := vm.WaitforPods(helpers.CiliumNamespace, "-l k8s-app=cilium-pre-flight-check", longTimeout)
	warningMessage := ""
	if err != nil {
		res := vm.Exec(fmt.Sprintf(
			"%s -n %s get pods -l k8s-app=cilium-pre-flight-check",
			helpers.KubectlCmd, helpers.CiliumNamespace))
		warningMessage = res.Stdout()
	}
	Expect(err).To(BeNil(), "cilium pre-flight check is not ready after timeout, pods status:\n %s", warningMessage)
}

// DeployCiliumAndDNS deploys DNS and cilium into the kubernetes cluster
func DeployCiliumAndDNS(vm *helpers.Kubectl, ciliumFilename string) {
	DeployCiliumOptionsAndDNS(vm, ciliumFilename, map[string]string{"global.debug.verbose": "flow"})
}

func redeployCilium(vm *helpers.Kubectl, ciliumFilename string, options map[string]string) {
	By("Installing Cilium")
	err := vm.CiliumInstall(ciliumFilename, options)
	Expect(err).To(BeNil(), "Cilium cannot be installed")

	err = vm.WaitForCiliumReadiness()
	Expect(err).To(BeNil(), "Timeout while waiting for Cilium to become ready")
}

// RedeployCilium reinstantiates the Cilium DS and ensures it is running.
//
// This helper is only appropriate for reconfiguring Cilium in the middle of
// an existing testsuite that calls DeployCiliumAndDNS(...).
func RedeployCilium(vm *helpers.Kubectl, ciliumFilename string, options map[string]string) {
	redeployCilium(vm, ciliumFilename, options)
	err := vm.CiliumPreFlightCheck()
	ExpectWithOffset(1, err).Should(BeNil(), "cilium pre-flight checks failed")
	ExpectCiliumOperatorReady(vm)
}

// UninstallCiliumFromManifest uninstall a deployed Cilium configuration from the
// provided manifest file.
// Treat this as a cleanup function for RedeployCilium/Redeploy/DeployCiliumAndDNS/CiliumInstall.
func UninstallCiliumFromManifest(vm *helpers.Kubectl, ciliumFilename string) {
	By("Removing Cilium installation using generated helm manifest")

	Expect(vm.DeleteAndWait(ciliumFilename, true).GetError()).
		To(BeNil(), "Error removing cilium from installed manifest")
}

// RedeployCiliumWithMerge merges the configuration passed as "from" into
// "options", allowing the caller to preserve the previous Cilium
// configuration, along with passing new configuration. This function behaves
// equivalently to RedeployCilium. Note that "options" is deep copied, meaning
// it will NOT be modified. Any modifications will be local to this function.
func RedeployCiliumWithMerge(vm *helpers.Kubectl,
	ciliumFilename string,
	from, options map[string]string) {

	// Merge configuration
	newOpts := make(map[string]string, len(options))
	for k, v := range from {
		newOpts[k] = v
	}
	for k, v := range options {
		newOpts[k] = v
	}

	RedeployCilium(vm, ciliumFilename, newOpts)
}

// DeployCiliumOptionsAndDNS deploys DNS and cilium with options into the kubernetes cluster
func DeployCiliumOptionsAndDNS(vm *helpers.Kubectl, ciliumFilename string, options map[string]string) {
	redeployCilium(vm, ciliumFilename, options)

	vm.RedeployKubernetesDnsIfNecessary()

	switch helpers.GetCurrentIntegration() {
	case helpers.CIIntegrationGKE:
		vm.RestartUnmanagedPodsInNamespace(helpers.KubeSystemNamespace)
	}

	switch helpers.GetCurrentIntegration() {
	case helpers.CIIntegrationFlannel:
		By("Installing Flannel")
		vm.ApplyDefault(vm.GetFilePath("../examples/kubernetes/addons/flannel/flannel.yaml"))
	default:
	}

	err := vm.CiliumPreFlightCheck()
	ExpectWithOffset(1, err).Should(BeNil(), "cilium pre-flight checks failed")
	ExpectCiliumOperatorReady(vm)

	switch helpers.GetCurrentIntegration() {
	case helpers.CIIntegrationGKE:
		err := vm.WaitforPods(helpers.KubeSystemNamespace, "", longTimeout)
		ExpectWithOffset(1, err).Should(BeNil(), "kube-system pods were not able to get into ready state after restart")
	}
}

// SkipIfBenchmark will skip the test if benchmark is not specified
func SkipIfBenchmark() {
	if !config.CiliumTestConfig.Benchmarks {
		Skip("Benchmarks are skipped, specify -cilium.Benchmarks")
	}
}

// SkipIfIntegration will skip a test if it's running with any of the specified
// integration.
func SkipIfIntegration(integration string) {
	if helpers.IsIntegration(integration) {
		Skip(fmt.Sprintf(
			"This feature is not supported in Cilium %q mode. Skipping test.",
			integration))
	}
}

// SkipItIfNoKubeProxy will skip It if kube-proxy is disabled (= NodePort BPF is
// enabled)
func SkipItIfNoKubeProxy() {
	if !helpers.RunsWithKubeProxy() {
		Skip("kube-proxy is disabled (NodePort BPF is enabled). Skipping test.")
	}
}
<commit_msg>ci/gke: Restart unmanaged pods in Cilium namespace<commit_after>// Copyright 2018-2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"
	"time"

	"github.com/cilium/cilium/test/config"
	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/gomega"
)

var longTimeout = 10 * time.Minute

// ExpectKubeDNSReady is a wrapper around helpers/WaitKubeDNS. It asserts that
// the error returned by that function is nil.
func ExpectKubeDNSReady(vm *helpers.Kubectl) {
	By("Waiting for kube-dns to be ready")
	err := vm.WaitKubeDNS()
	ExpectWithOffset(1, err).Should(BeNil(), "kube-dns was not able to get into ready state")

	By("Running kube-dns preflight check")
	err = vm.KubeDNSPreFlightCheck()
	ExpectWithOffset(1, err).Should(BeNil(), "kube-dns service not ready")
}

// ExpectCiliumReady is a wrapper around helpers/WaitForPods. It asserts that
// the error returned by that function is nil.
func ExpectCiliumReady(vm *helpers.Kubectl) {
	err := vm.WaitForCiliumReadiness()
	Expect(err).To(BeNil(), "Timeout while waiting for Cilium to become ready")

	err = vm.CiliumPreFlightCheck()
	ExpectWithOffset(1, err).Should(BeNil(), "cilium pre-flight checks failed")
}

// ExpectCiliumOperatorReady is a wrapper around helpers/WaitForPods. It asserts that
// the error returned by that function is nil.
func ExpectCiliumOperatorReady(vm *helpers.Kubectl) {
	By("Waiting for cilium-operator to be ready")
	err := vm.WaitforPods(helpers.CiliumNamespace, "-l name=cilium-operator", longTimeout)
	ExpectWithOffset(1, err).Should(BeNil(), "Cilium operator was not able to get into ready state")
}

// ExpectHubbleCLIReady is a wrapper around helpers/WaitForPods. It asserts
// that the error returned by that function is nil.
func ExpectHubbleCLIReady(vm *helpers.Kubectl, ns string) {
	By("Waiting for hubble-cli to be ready")
	err := vm.WaitforPods(ns, "-l k8s-app=hubble-cli", longTimeout)
	ExpectWithOffset(1, err).Should(BeNil(), "hubble-cli was not able to get into ready state")
}

// ExpectHubbleRelayReady is a wrapper around helpers/WaitForPods. It asserts
// that the error returned by that function is nil.
func ExpectHubbleRelayReady(vm *helpers.Kubectl, ns string) {
	By("Waiting for hubble-relay to be ready")
	err := vm.WaitforPods(ns, "-l k8s-app=hubble-relay", longTimeout)
	ExpectWithOffset(1, err).Should(BeNil(), "hubble-relay was not able to get into ready state")
}

// ExpectAllPodsTerminated is a wrapper around helpers/WaitCleanAllTerminatingPods.
// It asserts that the error returned by that function is nil.
func ExpectAllPodsTerminated(vm *helpers.Kubectl) {
	err := vm.WaitCleanAllTerminatingPods(helpers.HelperTimeout)
	ExpectWithOffset(1, err).To(BeNil(), "terminating containers are not deleted after timeout")
}

// ExpectAllPodsInNsTerminated is a wrapper around helpers/WaitCleanAllTerminatingPods.
// It asserts that the error returned by that function is nil.
func ExpectAllPodsInNsTerminated(vm *helpers.Kubectl, ns string) {
	err := vm.WaitCleanAllTerminatingPodsInNs(ns, helpers.HelperTimeout)
	ExpectWithOffset(1, err).To(BeNil(), "terminating containers are not deleted after timeout")
}

// ExpectCiliumPreFlightInstallReady is a wrapper around helpers/WaitForNPods.
// It asserts the error returned by that function is nil.
func ExpectCiliumPreFlightInstallReady(vm *helpers.Kubectl) {
	By("Waiting for all cilium pre-flight pods to be ready")

	err := vm.WaitforPods(helpers.CiliumNamespace, "-l k8s-app=cilium-pre-flight-check", longTimeout)
	warningMessage := ""
	if err != nil {
		res := vm.Exec(fmt.Sprintf(
			"%s -n %s get pods -l k8s-app=cilium-pre-flight-check",
			helpers.KubectlCmd, helpers.CiliumNamespace))
		warningMessage = res.Stdout()
	}
	Expect(err).To(BeNil(), "cilium pre-flight check is not ready after timeout, pods status:\n %s", warningMessage)
}

// DeployCiliumAndDNS deploys DNS and cilium into the kubernetes cluster
func DeployCiliumAndDNS(vm *helpers.Kubectl, ciliumFilename string) {
	DeployCiliumOptionsAndDNS(vm, ciliumFilename, map[string]string{"global.debug.verbose": "flow"})
}

func redeployCilium(vm *helpers.Kubectl, ciliumFilename string, options map[string]string) {
	By("Installing Cilium")
	err := vm.CiliumInstall(ciliumFilename, options)
	Expect(err).To(BeNil(), "Cilium cannot be installed")

	err = vm.WaitForCiliumReadiness()
	Expect(err).To(BeNil(), "Timeout while waiting for Cilium to become ready")
}

// RedeployCilium reinstantiates the Cilium DS and ensures it is running.
//
// This helper is only appropriate for reconfiguring Cilium in the middle of
// an existing testsuite that calls DeployCiliumAndDNS(...).
func RedeployCilium(vm *helpers.Kubectl, ciliumFilename string, options map[string]string) {
	redeployCilium(vm, ciliumFilename, options)
	err := vm.CiliumPreFlightCheck()
	ExpectWithOffset(1, err).Should(BeNil(), "cilium pre-flight checks failed")
	ExpectCiliumOperatorReady(vm)
}

// UninstallCiliumFromManifest uninstall a deployed Cilium configuration from the
// provided manifest file.
// Treat this as a cleanup function for RedeployCilium/Redeploy/DeployCiliumAndDNS/CiliumInstall.
func UninstallCiliumFromManifest(vm *helpers.Kubectl, ciliumFilename string) {
	By("Removing Cilium installation using generated helm manifest")

	Expect(vm.DeleteAndWait(ciliumFilename, true).GetError()).
		To(BeNil(), "Error removing cilium from installed manifest")
}

// RedeployCiliumWithMerge merges the configuration passed as "from" into
// "options", allowing the caller to preserve the previous Cilium
// configuration, along with passing new configuration. This function behaves
// equivalently to RedeployCilium. Note that "options" is deep copied, meaning
// it will NOT be modified. Any modifications will be local to this function.
func RedeployCiliumWithMerge(vm *helpers.Kubectl,
	ciliumFilename string,
	from, options map[string]string) {

	// Merge configuration
	newOpts := make(map[string]string, len(options))
	for k, v := range from {
		newOpts[k] = v
	}
	for k, v := range options {
		newOpts[k] = v
	}

	RedeployCilium(vm, ciliumFilename, newOpts)
}

// DeployCiliumOptionsAndDNS deploys DNS and cilium with options into the kubernetes cluster
func DeployCiliumOptionsAndDNS(vm *helpers.Kubectl, ciliumFilename string, options map[string]string) {
	redeployCilium(vm, ciliumFilename, options)

	vm.RedeployKubernetesDnsIfNecessary()

	switch helpers.GetCurrentIntegration() {
	case helpers.CIIntegrationGKE:
		vm.RestartUnmanagedPodsInNamespace(helpers.KubeSystemNamespace)
		vm.RestartUnmanagedPodsInNamespace(helpers.CiliumNamespace)
	}

	switch helpers.GetCurrentIntegration() {
	case helpers.CIIntegrationFlannel:
		By("Installing Flannel")
		vm.ApplyDefault(vm.GetFilePath("../examples/kubernetes/addons/flannel/flannel.yaml"))
	default:
	}

	err := vm.CiliumPreFlightCheck()
	ExpectWithOffset(1, err).Should(BeNil(), "cilium pre-flight checks failed")
	ExpectCiliumOperatorReady(vm)

	switch helpers.GetCurrentIntegration() {
	case helpers.CIIntegrationGKE:
		err := vm.WaitforPods(helpers.KubeSystemNamespace, "", longTimeout)
		ExpectWithOffset(1, err).Should(BeNil(), "kube-system pods were not able to get into ready state after restart")
	}
}

// SkipIfBenchmark will skip the test if benchmark is not specified
func SkipIfBenchmark() {
	if !config.CiliumTestConfig.Benchmarks {
		Skip("Benchmarks are skipped, specify -cilium.Benchmarks")
	}
}

// SkipIfIntegration will skip a test if it's running with any of the specified
// integration.
func SkipIfIntegration(integration string) {
	if helpers.IsIntegration(integration) {
		Skip(fmt.Sprintf(
			"This feature is not supported in Cilium %q mode. Skipping test.",
			integration))
	}
}

// SkipItIfNoKubeProxy will skip It if kube-proxy is disabled (= NodePort BPF is
// enabled)
func SkipItIfNoKubeProxy() {
	if !helpers.RunsWithKubeProxy() {
		Skip("kube-proxy is disabled (NodePort BPF is enabled). Skipping test.")
	}
}
<|endoftext|>


<commit_before>package polynomial

import (
	"math"
)

// Open is a basis in (0, 1)^n.
type Open struct {
	nd uint
}

// NewOpen creates a basis in (0, 1)^n.
func NewOpen(dimensions, order uint) *Open {
	if order != 1 {
		panic("not implemented")
	}
	return &Open{dimensions}
}

// Compute evaluates a basis function.
func (self *Open) Compute(index []uint64, point []float64) float64 {
	nd := self.nd

	value := 1.0
	for i := uint(0); i < nd; i++ {
		level := levelMask & index[i]
		if level == 0 {
			continue // value *= 1.0
		}

		order := index[i] >> levelSize
		count := uint64(2)<<level - 1

		x := point[i]
		switch order {
		case 0:
			step := 1.0 / float64(count+1)
			if x >= 2.0*step {
				return 0.0 // value *= 0.0
			}
			value *= 2.0 - x/step
		case count - 1:
			step, left := 1.0/float64(count+1), float64(count-1)
			if x <= left*step {
				return 0.0 // value *= 0.0
			}
			value *= x/step - left
		default:
			xi, step := openNode(level, order)
			delta := math.Abs(x - xi)
			if delta >= step {
				return 0.0 // value *= 0.0
			}
			value *= 1.0 - delta/step
		}
	}

	return value
}

// Integrate computes the integral of a basis function.
func (self *Open) Integrate(index []uint64) float64 {
	nd := self.nd

	value := 1.0
	for i := uint(0); i < nd; i++ {
		level := levelMask & index[i]
		if level == 0 {
			continue // value *= 1.0
		}

		order := index[i] >> levelSize
		count := uint64(2)<<level - 1

		switch order {
		case 0, count - 1:
			value *= 2.0 / float64(count+1)
		default:
			value *= 1.0 / float64(count+1)
		}
	}

	return value
}

func openNode(level, order uint64) (x, step float64) {
	step = 1.0 / float64(uint64(2)<<level)
	x = float64(order+1) * step
	return
}
<commit_msg>b/polynomial: make Open match Closed<commit_after>package polynomial

import (
	"math"
)

// Open is a basis in (0, 1)^n.
type Open struct {
	nd uint
}

// NewOpen creates a basis in (0, 1)^n.
func NewOpen(dimensions, order uint) *Open {
	if order != 1 {
		panic("not implemented")
	}
	return &Open{dimensions}
}

// Compute evaluates a basis function.
func (self *Open) Compute(index []uint64, point []float64) float64 {
	nd, value := self.nd, 1.0
	for i := uint(0); i < nd && value != 0.0; i++ {
		value *= openCompute(levelMask&index[i], index[i]>>levelSize, point[i])
	}
	return value
}

// Integrate computes the integral of a basis function.
func (self *Open) Integrate(index []uint64) float64 {
	nd, value := self.nd, 1.0
	for i := uint(0); i < nd && value != 0.0; i++ {
		value *= openIntegrate(levelMask&index[i], index[i]>>levelSize)
	}
	return value
}

func openCompute(level, order uint64, x float64) float64 {
	if level == 0 {
		return 1.0
	}

	count := uint64(2)<<level - 1

	switch order {
	case 0:
		step := 1.0 / float64(count+1)
		if x >= 2.0*step {
			return 0.0
		}
		return 2.0 - x/step
	case count - 1:
		step, left := 1.0/float64(count+1), float64(count-1)
		if x <= left*step {
			return 0.0
		}
		return x/step - left
	default:
		xi, step := openNode(level, order)
		delta := math.Abs(x - xi)
		if delta >= step {
			return 0.0
		}
		return 1.0 - delta/step
	}
}

func openIntegrate(level, order uint64) float64 {
	if level == 0 {
		return 1.0
	}

	count := uint64(2)<<level - 1

	switch order {
	case 0, count - 1:
		return 2.0 / float64(count+1)
	default:
		return 1.0 / float64(count+1)
	}
}

func openNode(level, order uint64) (x, step float64) {
	step = 1.0 / float64(uint64(2)<<level)
	x = float64(order+1) * step
	return
}
<|endoftext|>


<commit_before>package main

// Usage: go run correos.go <trackid>

import (
	"encoding/xml"
	"fmt"
	"golang.org/x/text/encoding/charmap"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

type Response struct {
	XMLOut string `xml:"Body>ConsultaLocalizacionEnviosFasesResponse>ConsultaLocalizacionEnviosFasesResult"`
}

type ConsultaXMLout struct {
	Events []Event `xml:"Respuestas>DatosIdiomas>DatosEnvios>Datos"`
}

type Event struct {
	Status       string `xml:"Estado"`
	Date         string `xml:"Fecha"`
	TrackingCode string `xml:"Codigo,attr"`
	Event        int    `xml:"Evento,attr"`
}

func makeCharsetReader(charset string, input io.Reader) (io.Reader, error) {
	switch charset {
	case "ISO-8859-1":
		// Windows-1252 is a superset of ISO-8859-1, so should do here
	case "Windows-1252":

		return charmap.Windows1252.NewDecoder().Reader(input), nil
	default:
		return nil, fmt.Errorf("Unknown charset: %s", charset)
	}
	return nil, fmt.Errorf("Unknown charset: %s", charset)
}
func main() {
	if len(os.Args) < 2 {
		fmt.Println("Usage: correos <tracknumber>")
		os.Exit(1)
	}
	shipmentCode := os.Args[1]
	// http://www.correos.es/ss/Satellite/site/aplicacion-magento-de_integracion/detalle_app-sidioma=es_ES
	xmlSend := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<soap:Body>
<ConsultaLocalizacionEnviosFases xmlns="ServiciosWebLocalizacionMI/">
<XMLin><![CDATA[<?xml version="1.0" encoding="utf-8" ?><ConsultaXMLin Idioma="1" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><Consulta><Codigo>%s</Codigo></Consulta></ConsultaXMLin>]]></XMLin>
</ConsultaLocalizacionEnviosFases>
</soap:Body>
</soap:Envelope>`, shipmentCode)

	// https://play.golang.org/p/lpht7gSN3k by Henrik-Johansson
	req, err := http.NewRequest("POST", "https://online.correos.es/servicioswebLocalizacionMI/localizacionMI.asmx", strings.NewReader(xmlSend))
	req.Method = "POST"
	//	fmt.Println(req)
	if err != nil {
		fmt.Println(err)
		return
	}

	req.Header.Add("ContentType", "text/xml;charset=UTF-8")
	req.Header.Add("SOAPAction", "")
	client := http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}

	b, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		fmt.Println(err)
		return
	}

	//	fmt.Println(string(b))
	// https://stackoverflow.com/a/23634278/267777
	res := &Response{}
	err = xml.Unmarshal(b, res)
	fmt.Println(res.XMLOut, err)

	// https://stackoverflow.com/a/34712322/267777
	res2 := &ConsultaXMLout{}
	decoder := xml.NewDecoder(strings.NewReader(res.XMLOut))
	decoder.CharsetReader = makeCharsetReader
	err = decoder.Decode(&res2)
	//	err = xml.Unmarshal([]byte(res.Body.GetResponse.XMLOut), res2)
	for _, v := range res2.Events {
		fmt.Printf("%s: %d %s\n", v.Date, v.Event, v.Status)
	}
}
<commit_msg>Another commented out debug line<commit_after>package main

// Usage: go run correos.go <trackid>

import (
	"encoding/xml"
	"fmt"
	"golang.org/x/text/encoding/charmap"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

type Response struct {
	XMLOut string `xml:"Body>ConsultaLocalizacionEnviosFasesResponse>ConsultaLocalizacionEnviosFasesResult"`
}

type ConsultaXMLout struct {
	Events []Event `xml:"Respuestas>DatosIdiomas>DatosEnvios>Datos"`
}

type Event struct {
	Status       string `xml:"Estado"`
	Date         string `xml:"Fecha"`
	TrackingCode string `xml:"Codigo,attr"`
	Event        int    `xml:"Evento,attr"`
}

func makeCharsetReader(charset string, input io.Reader) (io.Reader, error) {
	switch charset {
	case "ISO-8859-1":
		// Windows-1252 is a superset of ISO-8859-1, so should do here
	case "Windows-1252":

		return charmap.Windows1252.NewDecoder().Reader(input), nil
	default:
		return nil, fmt.Errorf("Unknown charset: %s", charset)
	}
	return nil, fmt.Errorf("Unknown charset: %s", charset)
}
func main() {
	if len(os.Args) < 2 {
		fmt.Println("Usage: correos <tracknumber>")
		os.Exit(1)
	}
	shipmentCode := os.Args[1]
	// http://www.correos.es/ss/Satellite/site/aplicacion-magento-de_integracion/detalle_app-sidioma=es_ES
	xmlSend := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<soap:Body>
<ConsultaLocalizacionEnviosFases xmlns="ServiciosWebLocalizacionMI/">
<XMLin><![CDATA[<?xml version="1.0" encoding="utf-8" ?><ConsultaXMLin Idioma="1" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><Consulta><Codigo>%s</Codigo></Consulta></ConsultaXMLin>]]></XMLin>
</ConsultaLocalizacionEnviosFases>
</soap:Body>
</soap:Envelope>`, shipmentCode)

	// https://play.golang.org/p/lpht7gSN3k by Henrik-Johansson
	req, err := http.NewRequest("POST", "https://online.correos.es/servicioswebLocalizacionMI/localizacionMI.asmx", strings.NewReader(xmlSend))
	req.Method = "POST"
	//	fmt.Println(req)
	if err != nil {
		fmt.Println(err)
		return
	}

	req.Header.Add("ContentType", "text/xml;charset=UTF-8")
	req.Header.Add("SOAPAction", "")
	client := http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}

	b, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		fmt.Println(err)
		return
	}

	//	fmt.Println(string(b))
	// https://stackoverflow.com/a/23634278/267777
	res := &Response{}
	err = xml.Unmarshal(b, res)
	//	fmt.Println(res.XMLOut, err)

	// https://stackoverflow.com/a/34712322/267777
	res2 := &ConsultaXMLout{}
	decoder := xml.NewDecoder(strings.NewReader(res.XMLOut))
	decoder.CharsetReader = makeCharsetReader
	err = decoder.Decode(&res2)
	//	err = xml.Unmarshal([]byte(res.Body.GetResponse.XMLOut), res2)
	for _, v := range res2.Events {
		fmt.Printf("%s: %d %s\n", v.Date, v.Event, v.Status)
	}
}
<|endoftext|>